From d0887f32345f34ebc2828751edf299be5f643ea7 Mon Sep 17 00:00:00 2001 From: ediazrod Date: Thu, 26 Feb 2015 18:48:31 +0100 Subject: [PATCH 001/143] This is a especific configuration for shorewall ipset proto6 Use ipset proto6 in shorewall. You must follow the rules to enable ipset in you blacklist if you have a lot of spam (my case) is better use ipset rather than shorewall command line (is my firewall) stop fail2ban with shorewall on one list of 1000 Ips takes 5 min with ipset in shorewall 10 sec. --- config/action.d/shorewall-ipset-proto6.conf | 84 +++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 config/action.d/shorewall-ipset-proto6.conf diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf new file mode 100644 index 00000000..b18fda6d --- /dev/null +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -0,0 +1,84 @@ +# Fail2Ban configuration file +# +# Author: Eduardo Diaz +# +# This is for ipset protocol 6 (and hopefully later) (ipset v6.14). +# for shorewall +# +# Use this setting in jail.conf to modify the name and the max time in every jain +# action = shorewall-ipset-proto6[name=SSH, bantime=10000] +# +# This requires the program ipset which is normally in package called ipset. +# +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# +# The default Shorewall configuration is with "BLACKLISTNEWONLY=Yes" (see +# file /etc/shorewall/shorewall.conf). This means that when Fail2ban adds a +# new shorewall rule to ban an IP address, that rule will affect only new +# connections. So if the attempter goes on trying using the same connection +# he could even log in. In order to get the same behavior of the iptable +# action (so that the ban is immediate) the /etc/shorewall/shorewall.conf +# file should me modified with "BLACKLISTNEWONLY=No". +# +# A lot of this depends of the shorewall version at least have the 4.5.5 Version +# +# Enable to shorewall use a blacklist using iptables creating a file /etc/shorewall/blrules +# and adding "DROP net:+f2b-ssh all" one for every jail +# details in shorewall documentation blacklist. +# to enable restore you ipset You must set SAVE_IPSETS=Yes in shorewall.conf +# Is importan to read this documentation from shorewall http://shorewall.net/ipsets.html +# +# To force create the ipset in the case that somebody delete the ipset create a file +# /etc/shorewall/initdone and add one line for every ipset (this files are in Perl) +# take care of add 1; at the end of the file +# the example is: +# system("/usr/sbin/ipset -quiet -exist create f2b-ssh hash:ip timeout 600 "); +# 1; +# +# To destroy the ipset in shorewall you must add to the file /etc/shorewall/stopped +# # One line of every ipset +# system("/usr/sbin/ipset -quiet destroy f2b-ssh "); +# 1; # This must go to the end of the file if not shorewall complilation fails +# +# + + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = if ! ipset -quiet -name list f2b- >/dev/null; + then ipset -quiet -exist create f2b- hash:ip timeout ; + fi + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = ipset flush f2b- + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = ipset add f2b- timeout -exist + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = ipset del f2b- -exist + +[Init] + +# Option: bantime +# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Values: [ NUM ] Default: 600 +# +bantime = 600 From e26a1ad6b6a1cbdbb6e28e62af61da2ec5bd73bb Mon Sep 17 00:00:00 2001 From: ediazrod Date: Mon, 23 Mar 2015 00:55:06 +0100 Subject: [PATCH 002/143] Update shorewall-ipset-proto6.conf --- config/action.d/shorewall-ipset-proto6.conf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index b18fda6d..48308ec1 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -22,10 +22,9 @@ # # A lot of this depends of the shorewall version at least have the 4.5.5 Version # -# Enable to shorewall use a blacklist using iptables creating a file /etc/shorewall/blrules -# and adding "DROP net:+f2b-ssh all" one for every jail -# details in shorewall documentation blacklist. -# to enable restore you ipset You must set SAVE_IPSETS=Yes in shorewall.conf +# Enable shorewall to use a blacklist using iptables creating a file /etc/shorewall/blrules +# and adding "DROP net:+f2b-ssh all" one for every jail details in shorewall documentation blacklist. +# To enable restore you ipset You must set SAVE_IPSETS=Yes in shorewall.conf # Is importan to read this documentation from shorewall http://shorewall.net/ipsets.html # # To force create the ipset in the case that somebody delete the ipset create a file From 5fdd1d1ded22d15b13dc83fd5cbb1a0c02eeccbc Mon Sep 17 00:00:00 2001 From: ediazrod Date: Mon, 23 Mar 2015 00:56:37 +0100 Subject: [PATCH 003/143] Update shorewall-ipset-proto6.conf --- config/action.d/shorewall-ipset-proto6.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 48308ec1..7fbc21bb 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -20,7 +20,8 @@ # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf # file should me modified with "BLACKLISTNEWONLY=No". # -# A lot of this depends of the shorewall version at least have the 4.5.5 Version +# The use in IPset in shorewall depends of the version you must have at least 4.5.5< version +# of shorewall # # Enable shorewall to use a blacklist using iptables creating a file /etc/shorewall/blrules # and adding "DROP net:+f2b-ssh all" one for every jail details in shorewall documentation blacklist. @@ -37,7 +38,7 @@ # To destroy the ipset in shorewall you must add to the file /etc/shorewall/stopped # # One line of every ipset # system("/usr/sbin/ipset -quiet destroy f2b-ssh "); -# 1; # This must go to the end of the file if not shorewall complilation fails +# 1; # This must go to the end of the file if not shorewall compilation fails # # From c9b24839e4338059224dcc743eb7f458f850f18e Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 14:27:41 +0100 Subject: [PATCH 004/143] Character detection heuristics for whois output via optional setting in mail-whois*.conf (Closes #1003) when set by user, - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command - converts whois data to UTF-8 character set with iconv - sends the whois output in UTF-8 character set to mail program - avoids that heirloom mailx creates binary attachment for input with unknown character set --- ChangeLog | 5 +++++ config/action.d/mail-whois-common.conf | 24 ++++++++++++++++++++++++ config/action.d/mail-whois-lines.conf | 6 +++++- config/action.d/mail-whois.conf | 6 +++++- 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 config/action.d/mail-whois-common.conf diff --git a/ChangeLog b/ChangeLog index 19eacf83..11365ee6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -78,6 +78,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * Absorbed DNSUtils.cidr into addr2bin in filter.py, added unittests * Added syslogsocket configuration to fail2ban.conf * Note in the jail.conf for the recidive jail to increase dbpurgeage (gh-964) + * Character detection heuristics for whois output via optional setting in mail-whois*.conf. When set by user, + - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command + - converts whois data to UTF-8 character set with iconv + - sends the whois output in UTF-8 character set to mail program + - avoids that heirloom mailx creates binary attachment for input with unknown character set ver. 0.9.1 (2014/10/29) - better, faster, stronger diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf new file mode 100644 index 00000000..77f49099 --- /dev/null +++ b/config/action.d/mail-whois-common.conf @@ -0,0 +1,24 @@ +# Fail2Ban configuration file +# +# Common settings for mail actions +# +# Users can override the defaults in mail-whois-common.local + +[INCLUDES] + +# Load customizations if any available +after = mail-whois-common.local + +[DEFAULT] +#original character set of whois output will be sent to mail program +_whois = whois || echo "missing whois program" + +# use heuristics to convert charset of whois output to a target +# character set before sending it to a mail program +_whois_target_charset = UTF-8 +_whois_convert_charset = whois | + { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + +# choose between _whois and _whois_convert_charset in mail-whois-common.local +_whois_command = %(_whois)s +#_whois_command = %(_whois_convert_charset)s diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 5f760ac8..35abeea1 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -4,6 +4,10 @@ # Modified-By: Yaroslav Halchenko to include grepping on IP over log files # +[INCLUDES] + +before = mail-whois-common.conf + [Definition] # Option: actionstart @@ -40,7 +44,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois || echo missing whois program`\n\n + `%(_whois_command)s`\n\n Lines containing IP: in \n `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index e4c8450e..fe81f499 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = mail-whois-common.conf + [Definition] # Option: actionstart @@ -40,7 +44,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois || echo missing whois program`\n + `%(_whois_command)s`\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" From c0cf3daac81ff95bdb18f1293ccb9f72ac6d0ebc Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 18:20:25 +0100 Subject: [PATCH 005/143] Add myself to the changelog --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 11365ee6..2522b9a4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -78,7 +78,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * Absorbed DNSUtils.cidr into addr2bin in filter.py, added unittests * Added syslogsocket configuration to fail2ban.conf * Note in the jail.conf for the recidive jail to increase dbpurgeage (gh-964) - * Character detection heuristics for whois output via optional setting in mail-whois*.conf. When set by user, + * Character detection heuristics for whois output via optional setting in mail-whois*.conf. Thanks Thomas Mayer + When set by user, - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command - converts whois data to UTF-8 character set with iconv - sends the whois output in UTF-8 character set to mail program From 80f11a4d28d3cdf91a217517f9130c3e6fb63473 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 18:36:09 +0100 Subject: [PATCH 006/143] Add empty Init Section to pass tests (issue #1003) --- config/action.d/mail-whois-common.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 77f49099..178081e4 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -20,5 +20,8 @@ _whois_convert_charset = whois | { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local +# or other *.local which include mail-whois-common.conf. _whois_command = %(_whois)s #_whois_command = %(_whois_convert_charset)s + +[Init] From 1c2e8f418575463ac9295d39e3f3743453bc68f8 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 19:12:05 +0100 Subject: [PATCH 007/143] Add myself to the THANKS section --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 5482b21d..2c107394 100644 --- a/THANKS +++ b/THANKS @@ -108,6 +108,7 @@ Stefan Tatschner Stephen Gildea Steven Hiscocks TESTOVIK +Thomas Mayer Tom Pike Tomas Pihl Tony Lawrence From 4a598070c8eebb2062c241b3361742db1911c265 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sat, 28 Mar 2015 06:58:01 +0100 Subject: [PATCH 008/143] remove '-ne' option as it's not interpreted any way (issue #1003) --- config/action.d/mail-whois-common.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 178081e4..b0de00e0 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,9 +15,10 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program +# make sure you have 'file' and 'iconv' commands installed when using this _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(echo "$c" | file -b --mime-encoding -) ; echo "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From ac1e41ea703cda2bfd6be064ed469c65ea5579ae Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 17:54:25 +0200 Subject: [PATCH 009/143] Revert "remove '-ne' option as it's not interpreted any way (issue #1003)" This reverts commit 4a598070c8eebb2062c241b3361742db1911c265. --- config/action.d/mail-whois-common.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index b0de00e0..178081e4 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,10 +15,9 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program -# make sure you have 'file' and 'iconv' commands installed when using this _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo "$c" | file -b --mime-encoding -) ; echo "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From 675c3a7c95b7427f3f6517302d5306f488852637 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 18:08:47 +0200 Subject: [PATCH 010/143] use printf instead of echo for POSIX compatibility (issue #1003) --- config/action.d/mail-whois-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 178081e4..1da54286 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -17,7 +17,7 @@ _whois = whois || echo "missing whois program" # character set before sending it to a mail program _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(printf %%b "$c" | file -b --mime-encoding -) ; printf %%b "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From 923d807ef866fc23656708588e5a04644f178f36 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 18:18:30 +0200 Subject: [PATCH 011/143] use human-readable variable names (issue #1003) --- config/action.d/mail-whois-common.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 1da54286..b0d27afc 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,9 +15,10 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program +# make sure you have 'file' and 'iconv' commands installed when opting for that _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(printf %%b "$c" | file -b --mime-encoding -) ; printf %%b "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { WHOIS_OUTPUT=$(cat) ; WHOIS_CHARSET=$(printf %%b "$WHOIS_OUTPUT" | file -b --mime-encoding -) ; printf %%b "$WHOIS_OUTPUT" | iconv -f $WHOIS_CHARSET -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From 6d72602f6b25aee10a3e0744635d3090a6415518 Mon Sep 17 00:00:00 2001 From: "M. Maraun" Date: Fri, 3 Jul 2015 22:42:22 +0200 Subject: [PATCH 012/143] Set Timeout at urlopen to 3 seconds --- THANKS | 1 + config/action.d/badips.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 5ae86a3c..133bd987 100644 --- a/THANKS +++ b/THANKS @@ -71,6 +71,7 @@ kojiro Lars Kneschke Lee Clemens leftyfb (Mike Rushton) +M. Maraun Manuel Arostegui Ramirez Marcel Dopita Mark Edgington diff --git a/config/action.d/badips.py b/config/action.d/badips.py index c2a239f5..58e64990 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -116,7 +116,7 @@ class BadIPsAction(ActionBase): """ try: response = urlopen( - self._Request("/".join([self._badips, "get", "categories"]))) + self._Request("/".join([self._badips, "get", "categories"])), None, 3) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) self._logSys.error( From 5d60700c0ca0910275629db956caffb117618cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Fri, 10 Jul 2015 16:22:43 +0200 Subject: [PATCH 013/143] Added pass2allow (knocking with fail2ban) --- ChangeLog | 3 + config/action.d/allow-iptables-multiport.conf | 59 +++++++++++++++++++ config/filter.d/apache-pass.conf | 20 +++++++ config/jail.conf | 13 ++++ 4 files changed, 95 insertions(+) create mode 100644 config/action.d/allow-iptables-multiport.conf create mode 100644 config/filter.d/apache-pass.conf diff --git a/ChangeLog b/ChangeLog index bc849aaa..ac61693f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,9 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released - New Features: * New filters: - froxlor-auth Thanks Joern Muehlencord + * New type of operation: + - pass2allow: use fail2ban for "knocking", opening a closed port + (apache-pass filter, allow-iptables-multiport action) - Enhancements: * action.d/cloudflare.conf - improved documentation on how to allow diff --git a/config/action.d/allow-iptables-multiport.conf b/config/action.d/allow-iptables-multiport.conf new file mode 100644 index 00000000..6f9ffd71 --- /dev/null +++ b/config/action.d/allow-iptables-multiport.conf @@ -0,0 +1,59 @@ +# Fail2Ban configuration file for allowing hosts +# +# WARNING +# Please be aware that all users behind NAT will access the service on the specified port. +# You should protect this service with another jail that has very long bantime. + +[INCLUDES] + +before = iptables-common.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = iptables -N f2b- + iptables -A f2b- -j + iptables -I -p -m multiport --dports -j f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = iptables -D -p -m multiport --dports -j f2b- + iptables -F f2b- + iptables -X f2b- + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = iptables -I f2b- 1 -s -j + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = iptables -D f2b- -s -j + +[Init] + +# Option: allowtype +# Notes: ACCEPT skips other chains +# Value: [ RETURN | ACCEPT ] +# +allowtype = RETURN + +# Author: Viktor Szépe diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf new file mode 100644 index 00000000..dd00f953 --- /dev/null +++ b/config/filter.d/apache-pass.conf @@ -0,0 +1,20 @@ +# Fail2Ban Apache pass filter +# This filter is for access.log, NOT for error.log +# +# The knocking request must have a referer. + +[INCLUDES] + +before = apache-common.conf + +[Definition] + +failregex = ^ - \w+ \[\] "GET HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$ + +ignoreregex = + +[Init] + +knocking_url = /knocking/ + +# Author: Viktor Szépe diff --git a/config/jail.conf b/config/jail.conf index 67eda24e..ca0a2bfa 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -767,3 +767,16 @@ port = 2222 enabled = false logpath = /var/lib/portsentry/portsentry.history maxretry = 1 + +[pass2allow] +# allow FTP traffic after successful HTTP auth +enabled = false +filter = apache-pass +banaction = allow-iptables-multiport +# access log of the website with HTTP auth +logpath = /var/log/apache2/access.log +port = ftp,ftp-data,ftps,ftps-data +protocol = tcp +bantime = 3600 +maxretry = 1 +findtime = 1 From 5b7e1de2f4a99b00b70bb5f743780445cd8e2a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Sat, 11 Jul 2015 18:20:09 +0200 Subject: [PATCH 014/143] Instead of allow-iptables-multiport actions swap blocktype and (new) returntype --- config/action.d/allow-iptables-multiport.conf | 59 ------------------- config/action.d/iptables-allports.conf | 2 +- config/action.d/iptables-common.conf | 6 ++ config/action.d/iptables-multiport-log.conf | 2 +- config/action.d/iptables-multiport.conf | 2 +- config/action.d/iptables-new.conf | 2 +- config/action.d/iptables.conf | 2 +- config/jail.conf | 20 ++++--- 8 files changed, 22 insertions(+), 73 deletions(-) delete mode 100644 config/action.d/allow-iptables-multiport.conf diff --git a/config/action.d/allow-iptables-multiport.conf b/config/action.d/allow-iptables-multiport.conf deleted file mode 100644 index 6f9ffd71..00000000 --- a/config/action.d/allow-iptables-multiport.conf +++ /dev/null @@ -1,59 +0,0 @@ -# Fail2Ban configuration file for allowing hosts -# -# WARNING -# Please be aware that all users behind NAT will access the service on the specified port. -# You should protect this service with another jail that has very long bantime. - -[INCLUDES] - -before = iptables-common.conf - -[Definition] - -# Option: actionstart -# Notes.: command executed once at the start of Fail2Ban. -# Values: CMD -# -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I -p -m multiport --dports -j f2b- - -# Option: actionstop -# Notes.: command executed once at the end of Fail2Ban -# Values: CMD -# -actionstop = iptables -D -p -m multiport --dports -j f2b- - iptables -F f2b- - iptables -X f2b- - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = iptables -I f2b- 1 -s -j - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = iptables -D f2b- -s -j - -[Init] - -# Option: allowtype -# Notes: ACCEPT skips other chains -# Value: [ RETURN | ACCEPT ] -# -allowtype = RETURN - -# Author: Viktor Szépe diff --git a/config/action.d/iptables-allports.conf b/config/action.d/iptables-allports.conf index b30404d3..9e2d18a3 100644 --- a/config/action.d/iptables-allports.conf +++ b/config/action.d/iptables-allports.conf @@ -18,7 +18,7 @@ before = iptables-common.conf # Values: CMD # actionstart = iptables -N f2b- - iptables -A f2b- -j RETURN + iptables -A f2b- -j iptables -I -p -j f2b- # Option: actionstop diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf index c191c5a1..dff01362 100644 --- a/config/action.d/iptables-common.conf +++ b/config/action.d/iptables-common.conf @@ -43,3 +43,9 @@ protocol = tcp # REJECT, REJECT --reject-with icmp-port-unreachable # Values: STRING blocktype = REJECT --reject-with icmp-port-unreachable + +# Option: returntype +# Note: This is the default rule on "actionstart". This should be RETURN +# in all (blocking) actions, except REJECT in allowing actions. +# Values: STRING +returntype = RETURN diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf index f4d80d6c..093ce7b2 100644 --- a/config/action.d/iptables-multiport-log.conf +++ b/config/action.d/iptables-multiport-log.conf @@ -20,7 +20,7 @@ before = iptables-common.conf # Values: CMD # actionstart = iptables -N f2b- - iptables -A f2b- -j RETURN + iptables -A f2b- -j iptables -I 1 -p -m multiport --dports -j f2b- iptables -N f2b--log iptables -I f2b--log -j LOG --log-prefix "$(expr f2b- : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2 diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf index b70baf92..f365d917 100644 --- a/config/action.d/iptables-multiport.conf +++ b/config/action.d/iptables-multiport.conf @@ -15,7 +15,7 @@ before = iptables-common.conf # Values: CMD # actionstart = iptables -N f2b- - iptables -A f2b- -j RETURN + iptables -A f2b- -j iptables -I -p -m multiport --dports -j f2b- # Option: actionstop diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf index 3c6657d9..831931dd 100644 --- a/config/action.d/iptables-new.conf +++ b/config/action.d/iptables-new.conf @@ -17,7 +17,7 @@ before = iptables-common.conf # Values: CMD # actionstart = iptables -N f2b- - iptables -A f2b- -j RETURN + iptables -A f2b- -j iptables -I -m state --state NEW -p --dport -j f2b- # Option: actionstop diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf index a956fc55..572bdc11 100644 --- a/config/action.d/iptables.conf +++ b/config/action.d/iptables.conf @@ -15,7 +15,7 @@ before = iptables-common.conf # Values: CMD # actionstart = iptables -N f2b- - iptables -A f2b- -j RETURN + iptables -A f2b- -j iptables -I -p --dport -j f2b- # Option: actionstop diff --git a/config/jail.conf b/config/jail.conf index ca0a2bfa..c416c076 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -770,13 +770,15 @@ maxretry = 1 [pass2allow] # allow FTP traffic after successful HTTP auth -enabled = false -filter = apache-pass -banaction = allow-iptables-multiport +enabled = false +filter = apache-pass +banaction = iptables-multiport +blocktype = RETURN +returntype = DROP # access log of the website with HTTP auth -logpath = /var/log/apache2/access.log -port = ftp,ftp-data,ftps,ftps-data -protocol = tcp -bantime = 3600 -maxretry = 1 -findtime = 1 +logpath = /var/log/apache2/access.log +port = ftp,ftp-data,ftps,ftps-data +protocol = tcp +bantime = 3600 +maxretry = 1 +findtime = 1 From 586703dcc27ca2d98bb94bbeb4e6f0fce20909e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Mon, 13 Jul 2015 16:46:04 +0200 Subject: [PATCH 015/143] Test, changelog and fixes to pass2allow --- ChangeLog | 9 ++++++--- config/jail.conf | 9 +++------ fail2ban/tests/files/logs/apache-pass | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 fail2ban/tests/files/logs/apache-pass diff --git a/ChangeLog b/ChangeLog index ac61693f..de3de7f1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,11 +32,14 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released currently not banned or persistent) - New Features: + * RETURN iptables target is now a variable: + * New type of operation: pass2allow, use fail2ban for "knocking", + opening a closed port by swapping blocktype and returntype * New filters: - froxlor-auth Thanks Joern Muehlencord - * New type of operation: - - pass2allow: use fail2ban for "knocking", opening a closed port - (apache-pass filter, allow-iptables-multiport action) + - apache-pass - filter Apache access log for successfull authentication + * New jails: + - pass2allow-ftp - allows FTP traffic after successful HTTP authentication - Enhancements: * action.d/cloudflare.conf - improved documentation on how to allow diff --git a/config/jail.conf b/config/jail.conf index c416c076..350582fc 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -768,17 +768,14 @@ enabled = false logpath = /var/lib/portsentry/portsentry.history maxretry = 1 -[pass2allow] -# allow FTP traffic after successful HTTP auth -enabled = false +[pass2allow-ftp] +# this pass2allow example allows FTP traffic after successful HTTP authentication filter = apache-pass -banaction = iptables-multiport blocktype = RETURN returntype = DROP # access log of the website with HTTP auth -logpath = /var/log/apache2/access.log +logpath = %(apache_access_log)s port = ftp,ftp-data,ftps,ftps-data -protocol = tcp bantime = 3600 maxretry = 1 findtime = 1 diff --git a/fail2ban/tests/files/logs/apache-pass b/fail2ban/tests/files/logs/apache-pass new file mode 100644 index 00000000..cb8d3454 --- /dev/null +++ b/fail2ban/tests/files/logs/apache-pass @@ -0,0 +1,2 @@ +# failJSON: { "time": "2013-06-27T11:55:44", "match": true , "host": "192.0.2.12" } +192.0.2.12 - user1 [27/Jun/2013:11:55:44] "GET /knocking/ HTTP/1.1" 200 266 "http://domain.net/hello-world/" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:40.0) Gecko/20100101 Firefox/40.0" From b638e807ad818463859f4d0c1c774d01c621d133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Mon, 13 Jul 2015 18:12:04 +0200 Subject: [PATCH 016/143] Explicitly stating that knocking_url needs to be customized --- config/jail.conf | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 350582fc..a306e7ec 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -770,12 +770,14 @@ maxretry = 1 [pass2allow-ftp] # this pass2allow example allows FTP traffic after successful HTTP authentication -filter = apache-pass -blocktype = RETURN -returntype = DROP +port = ftp,ftp-data,ftps,ftps-data +filter = apache-pass # access log of the website with HTTP auth -logpath = %(apache_access_log)s -port = ftp,ftp-data,ftps,ftps-data -bantime = 3600 -maxretry = 1 -findtime = 1 +logpath = %(apache_access_log)s +# knocking URL needs to be customized per each deployment +knocking_url = /secret-knocking-url +blocktype = RETURN +returntype = DROP +bantime = 3600 +maxretry = 1 +findtime = 1 From 948b12e5df176ddee46cb2e03d4e316ff137c9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Tue, 14 Jul 2015 18:35:51 +0200 Subject: [PATCH 017/143] Fixed definition of knocking_url for pass2allow --- config/jail.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index a306e7ec..b0ed6355 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -771,11 +771,10 @@ maxretry = 1 [pass2allow-ftp] # this pass2allow example allows FTP traffic after successful HTTP authentication port = ftp,ftp-data,ftps,ftps-data +# knocking URL needs to be customized in apache-pass.local filter = apache-pass # access log of the website with HTTP auth logpath = %(apache_access_log)s -# knocking URL needs to be customized per each deployment -knocking_url = /secret-knocking-url blocktype = RETURN returntype = DROP bantime = 3600 From 7a011fca1b76f72fca55a3e386c064bc32b55228 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 16 Jul 2015 21:55:20 -0400 Subject: [PATCH 018/143] DOC: adjusted comment in pass2allow-ftp to my suggested wording --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index b0ed6355..f545ff13 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -771,7 +771,7 @@ maxretry = 1 [pass2allow-ftp] # this pass2allow example allows FTP traffic after successful HTTP authentication port = ftp,ftp-data,ftps,ftps-data -# knocking URL needs to be customized in apache-pass.local +# knocking_url variable must be overridden to some secret value in filter.d/apache-pass.local filter = apache-pass # access log of the website with HTTP auth logpath = %(apache_access_log)s From 31dc4e226383a24e6c2a31ca010ee1b18e21e437 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Jul 2015 21:29:05 -0400 Subject: [PATCH 019/143] ENH: added lockingopt option for iptables actions, made iptables cmd itself a parameter --- config/action.d/iptables-common.conf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf index dff01362..45819e3d 100644 --- a/config/action.d/iptables-common.conf +++ b/config/action.d/iptables-common.conf @@ -49,3 +49,16 @@ blocktype = REJECT --reject-with icmp-port-unreachable # in all (blocking) actions, except REJECT in allowing actions. # Values: STRING returntype = RETURN + +# Option: lockingopt +# Notes.: Option was introduced to iptables to prevent multiple instances from +# running concurrently and causing irratic behavior. -w was introduced +# in iptables 1.4.20, so might be absent on older systems +# See https://github.com/fail2ban/fail2ban/issues/1122 +# Values: STRING +lockingopt = -w + +# Option: iptables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +iptables = iptables From 916937bb6aaefa75bf75d0070ec1b0dcf27a5a04 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Jul 2015 21:38:10 -0400 Subject: [PATCH 020/143] RF: use to take effect of it being a parameter --- config/action.d/iptables-allports.conf | 18 ++++++------ config/action.d/iptables-common.conf | 2 +- config/action.d/iptables-ipset-proto4.conf | 4 +-- .../iptables-ipset-proto6-allports.conf | 4 +-- config/action.d/iptables-ipset-proto6.conf | 4 +-- config/action.d/iptables-multiport-log.conf | 28 +++++++++---------- config/action.d/iptables-multiport.conf | 18 ++++++------ config/action.d/iptables-new.conf | 18 ++++++------ config/action.d/iptables-xt_recent-echo.conf | 4 +-- config/action.d/iptables.conf | 18 ++++++------ .../symbiosis-blacklist-allports.conf | 6 ++-- 11 files changed, 62 insertions(+), 62 deletions(-) diff --git a/config/action.d/iptables-allports.conf b/config/action.d/iptables-allports.conf index 9e2d18a3..15f3cbcc 100644 --- a/config/action.d/iptables-allports.conf +++ b/config/action.d/iptables-allports.conf @@ -17,23 +17,23 @@ before = iptables-common.conf # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I -p -j f2b- +actionstart = -N f2b- + -A f2b- -j + -I -p -j f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p -j f2b- - iptables -F f2b- - iptables -X f2b- +actionstop = -D -p -j f2b- + -F f2b- + -X f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' +actioncheck = -n -L | grep -q 'f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -41,7 +41,7 @@ actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = iptables -I f2b- 1 -s -j +actionban = -I f2b- 1 -s -j # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -49,7 +49,7 @@ actionban = iptables -I f2b- 1 -s -j # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = iptables -D f2b- -s -j +actionunban = -D f2b- -s -j [Init] diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf index 45819e3d..89c5e371 100644 --- a/config/action.d/iptables-common.conf +++ b/config/action.d/iptables-common.conf @@ -61,4 +61,4 @@ lockingopt = -w # Option: iptables # Notes.: Actual command to be executed, including common to all calls options # Values: STRING -iptables = iptables +iptables = diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index c72b1a85..2f63cd4b 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -28,13 +28,13 @@ before = iptables-common.conf # Values: CMD # actionstart = ipset --create f2b- iphash - iptables -I -p -m multiport --dports -m set --match-set f2b- src -j + -I -p -m multiport --dports -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p -m multiport --dports -m set --match-set f2b- src -j +actionstop = -D -p -m multiport --dports -m set --match-set f2b- src -j ipset --flush f2b- ipset --destroy f2b- diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index aaeee461..1f1d336f 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -24,13 +24,13 @@ before = iptables-common.conf # Values: CMD # actionstart = ipset create f2b- hash:ip timeout - iptables -I -m set --match-set f2b- src -j + -I -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -m set --match-set f2b- src -j +actionstop = -D -m set --match-set f2b- src -j ipset flush f2b- ipset destroy f2b- diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index bd36c49e..3b51ef58 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -24,13 +24,13 @@ before = iptables-common.conf # Values: CMD # actionstart = ipset create f2b- hash:ip timeout - iptables -I -p -m multiport --dports -m set --match-set f2b- src -j + -I -p -m multiport --dports -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p -m multiport --dports -m set --match-set f2b- src -j +actionstop = -D -p -m multiport --dports -m set --match-set f2b- src -j ipset flush f2b- ipset destroy f2b- diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf index 093ce7b2..1777ce62 100644 --- a/config/action.d/iptables-multiport-log.conf +++ b/config/action.d/iptables-multiport-log.conf @@ -19,28 +19,28 @@ before = iptables-common.conf # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I 1 -p -m multiport --dports -j f2b- - iptables -N f2b--log - iptables -I f2b--log -j LOG --log-prefix "$(expr f2b- : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2 - iptables -A f2b--log -j +actionstart = -N f2b- + -A f2b- -j + -I 1 -p -m multiport --dports -j f2b- + -N f2b--log + -I f2b--log -j LOG --log-prefix "$(expr f2b- : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2 + -A f2b--log -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p -m multiport --dports -j f2b- - iptables -F f2b- - iptables -F f2b--log - iptables -X f2b- - iptables -X f2b--log +actionstop = -D -p -m multiport --dports -j f2b- + -F f2b- + -F f2b--log + -X f2b- + -X f2b--log # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L f2b--log >/dev/null +actioncheck = -n -L f2b--log >/dev/null # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -48,7 +48,7 @@ actioncheck = iptables -n -L f2b--log >/dev/null # Tags: See jail.conf(5) man page # Values: CMD # -actionban = iptables -I f2b- 1 -s -j f2b--log +actionban = -I f2b- 1 -s -j f2b--log # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -56,7 +56,7 @@ actionban = iptables -I f2b- 1 -s -j f2b--log # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = iptables -D f2b- -s -j f2b--log +actionunban = -D f2b- -s -j f2b--log [Init] diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf index f365d917..9fd87d20 100644 --- a/config/action.d/iptables-multiport.conf +++ b/config/action.d/iptables-multiport.conf @@ -14,23 +14,23 @@ before = iptables-common.conf # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I -p -m multiport --dports -j f2b- +actionstart = -N f2b- + -A f2b- -j + -I -p -m multiport --dports -j f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p -m multiport --dports -j f2b- - iptables -F f2b- - iptables -X f2b- +actionstop = -D -p -m multiport --dports -j f2b- + -F f2b- + -X f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' +actioncheck = -n -L | grep -q 'f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -38,7 +38,7 @@ actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = iptables -I f2b- 1 -s -j +actionban = -I f2b- 1 -s -j # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -46,7 +46,7 @@ actionban = iptables -I f2b- 1 -s -j # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = iptables -D f2b- -s -j +actionunban = -D f2b- -s -j [Init] diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf index 831931dd..795bc601 100644 --- a/config/action.d/iptables-new.conf +++ b/config/action.d/iptables-new.conf @@ -16,23 +16,23 @@ before = iptables-common.conf # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I -m state --state NEW -p --dport -j f2b- +actionstart = -N f2b- + -A f2b- -j + -I -m state --state NEW -p --dport -j f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -m state --state NEW -p --dport -j f2b- - iptables -F f2b- - iptables -X f2b- +actionstop = -D -m state --state NEW -p --dport -j f2b- + -F f2b- + -X f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' +actioncheck = -n -L | grep -q 'f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -40,7 +40,7 @@ actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = iptables -I f2b- 1 -s -j +actionban = -I f2b- 1 -s -j # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -48,7 +48,7 @@ actionban = iptables -I f2b- 1 -s -j # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = iptables -D f2b- -s -j +actionunban = -D f2b- -s -j [Init] diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf index 1a72968f..d3d43f86 100644 --- a/config/action.d/iptables-xt_recent-echo.conf +++ b/config/action.d/iptables-xt_recent-echo.conf @@ -32,14 +32,14 @@ before = iptables-common.conf # own rules. The 3600 second timeout is independent and acts as a # safeguard in case the fail2ban process dies unexpectedly. The # shorter of the two timeouts actually matters. -actionstart = if [ `id -u` -eq 0 ];then iptables -I -m recent --update --seconds 3600 --name f2b- -j ;fi +actionstart = if [ `id -u` -eq 0 ];then -I -m recent --update --seconds 3600 --name f2b- -j ;fi # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = echo / > /proc/net/xt_recent/f2b- - if [ `id -u` -eq 0 ];then iptables -D -m recent --update --seconds 3600 --name f2b- -j ;fi + if [ `id -u` -eq 0 ];then -D -m recent --update --seconds 3600 --name f2b- -j ;fi # Option: actioncheck # Notes.: command executed once before each actionban command diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf index 572bdc11..38985ffa 100644 --- a/config/action.d/iptables.conf +++ b/config/action.d/iptables.conf @@ -14,23 +14,23 @@ before = iptables-common.conf # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = iptables -N f2b- - iptables -A f2b- -j - iptables -I -p --dport -j f2b- +actionstart = -N f2b- + -A f2b- -j + -I -p --dport -j f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D -p --dport -j f2b- - iptables -F f2b- - iptables -X f2b- +actionstop = -D -p --dport -j f2b- + -F f2b- + -X f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' +actioncheck = -n -L | grep -q 'f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -38,7 +38,7 @@ actioncheck = iptables -n -L | grep -q 'f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = iptables -I f2b- 1 -s -j +actionban = -I f2b- 1 -s -j # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -46,7 +46,7 @@ actionban = iptables -I f2b- 1 -s -j # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = iptables -D f2b- -s -j +actionunban = -D f2b- -s -j [Init] diff --git a/config/action.d/symbiosis-blacklist-allports.conf b/config/action.d/symbiosis-blacklist-allports.conf index c4979302..d98767e7 100644 --- a/config/action.d/symbiosis-blacklist-allports.conf +++ b/config/action.d/symbiosis-blacklist-allports.conf @@ -22,21 +22,21 @@ actionstop = # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = iptables -n -L +actioncheck = -n -L # Option: actionban # Notes.: command executed when banning an IP. # Values: CMD # actionban = echo 'all' >| /etc/symbiosis/firewall/blacklist.d/.auto - iptables -I 1 -s -j + -I 1 -s -j # Option: actionunban # Notes.: command executed when unbanning an IP. # Values: CMD # actionunban = rm -f /etc/symbiosis/firewall/blacklist.d/.auto - iptables -D -s -j || : + -D -s -j || : [Init] From 749d3c160ce1c8f5ff22cbae4c6e12d273f67e54 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Jul 2015 21:53:37 -0400 Subject: [PATCH 021/143] BF: symbiosis-blacklist-allports now also requires iptables-common.conf --- config/action.d/symbiosis-blacklist-allports.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/action.d/symbiosis-blacklist-allports.conf b/config/action.d/symbiosis-blacklist-allports.conf index d98767e7..c24a8e0a 100644 --- a/config/action.d/symbiosis-blacklist-allports.conf +++ b/config/action.d/symbiosis-blacklist-allports.conf @@ -3,6 +3,9 @@ # Author: Yaroslav Halchenko # +[INCLUDES] + +before = iptables-common.conf [Definition] From ebdfbae5597995afeba304e40c50ac0e8e88b6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Fri, 24 Jul 2015 09:33:47 +0200 Subject: [PATCH 022/143] Added a space between IP address and the following colon --- ChangeLog | 2 ++ config/action.d/mail-whois-lines.conf | 2 +- config/action.d/mail-whois.conf | 2 +- config/action.d/sendmail-geoip-lines.conf | 2 +- config/action.d/sendmail-whois-ipjailmatches.conf | 2 +- config/action.d/sendmail-whois-ipmatches.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-matches.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- 9 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index de3de7f1..19df985b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -50,6 +50,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released * filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add HEAD method verb * Revamp of Travis and coverage automated testing + * Added a space between IP address and the following colon + in notification emails for easier text selection ver. 0.9.2 (2015/04/29) - better-quick-now-than-later diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 5f760ac8..598fd34d 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -39,7 +39,7 @@ actioncheck = actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `whois || echo missing whois program`\n\n Lines containing IP: in \n `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index e4c8450e..3b57e4e3 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -39,7 +39,7 @@ actioncheck = actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `whois || echo missing whois program`\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index 3a0c931e..929908db 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -26,7 +26,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n http://bgp.he.net/ip/ http://www.projecthoneypot.org/ip_ http://whois.domaintools.com/\n\n diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 3e97ca32..689ffe45 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -23,7 +23,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `/usr/bin/whois `\n\n Matches for with failures IP:\n \n\n diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index beafb48f..b06e6db6 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -23,7 +23,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `/usr/bin/whois `\n\n Matches with failures IP:\n \n\n diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index abbd60bc..663ec599 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -23,7 +23,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n Lines containing IP: in \n `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index f2a13745..8bca5937 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -23,7 +23,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `/usr/bin/whois `\n\n Matches:\n \n\n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index 31026ce4..55b80bc5 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -23,7 +23,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n + Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From 3c0d7f5a4c494d6ade9c2ca1c788f76a9b42e08e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 24 Jul 2015 11:59:53 -0400 Subject: [PATCH 023/143] BF: do not wrap iptables into itself. Thanks Lee --- config/action.d/iptables-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf index 89c5e371..45819e3d 100644 --- a/config/action.d/iptables-common.conf +++ b/config/action.d/iptables-common.conf @@ -61,4 +61,4 @@ lockingopt = -w # Option: iptables # Notes.: Actual command to be executed, including common to all calls options # Values: STRING -iptables = +iptables = iptables From 0ed1cb0aa6ba778f943f36cc190f08928e30e35e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 24 Jul 2015 13:57:59 -0400 Subject: [PATCH 024/143] Remove literal "TODO" from method's name Also need to change expected log message, since this test hasn't been executed, possibly ever. --- fail2ban/tests/clientreadertestcase.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 9275ae14..15066e7a 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -184,14 +184,20 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged("Found no accessible config files for 'filter.d/catchallthebadies' under %s" % IMPERFECT_CONFIG)) self.assertTrue(self._is_logged('Unable to read the filter')) - def TODOtestJailActionBrokenDef(self): - jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + def testJailActionBrokenDef(self): + jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, + share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) self.printLog() self.assertTrue(self._is_logged('Error in action definition joho[foo')) - self.assertTrue(self._is_logged('Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')) + # This unittest has been deactivated for some time... + # self.assertTrue(self._is_logged( + # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')) + # let's test for what is actually logged and handle changes in the future + self.assertTrue(self._is_logged( + "Caught exception: 'NoneType' object has no attribute 'endswith'")) if STOCK: def testStockSSHJail(self): From 8822ed4144e09ae071d77969f557808104fad3dc Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 24 Jul 2015 15:13:37 -0400 Subject: [PATCH 025/143] Remove self.printlog() call This seems to have been used for debugging, while unittest method was disabled --- fail2ban/tests/clientreadertestcase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 15066e7a..e3f1699a 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -190,7 +190,6 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.printLog() self.assertTrue(self._is_logged('Error in action definition joho[foo')) # This unittest has been deactivated for some time... # self.assertTrue(self._is_logged( From 33b204a2ee04a4059d633b004b9b1343c545cc08 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 26 Jul 2015 18:25:42 -0400 Subject: [PATCH 026/143] DOC: Changelog for iptables -w change --- ChangeLog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index de3de7f1..61992e40 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,13 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released - IMPORTANT incompatible changes: * filter.d/roundcube-auth.conf - Changed logpath to 'errors' log (was 'userlogins') + * action.d/iptables-common.conf + - All calls to iptables command now use -w switch introduced in + iptables 1.4.20 (some distribution could have patched their + earlier base version as well) to provide this locking mechanism + useful under heavy load to avoid contesting on iptables calls. + If you need to disable, define 'action.d/iptables-common.local' + with empty value for 'lockingopt' in `[Init]` section. - Fixes: * reload in interactive mode appends all the jails twice (gh-825) From 95478125261f6fa7437e4a26e9434dac686befd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Mon, 27 Jul 2015 01:05:05 +0200 Subject: [PATCH 027/143] ipjailmatches is on one line with its description in man jail.conf --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index db30251c..45eea040 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -279,7 +279,7 @@ concatenated string of the log file lines of the matches that generated the ban. .B ipmatches As per \fBmatches\fR, but includes all lines for the IP which are contained with the fail2ban persistent database. Therefore the database must be set for this tag to function. .TP -.B ipjailmatches\ +.B ipjailmatches As per \fBipmatches\fR, but matches are limited for the IP and for the current jail. .SH "PYTHON ACTION FILES" From c8b3ee10a01791b8c67eb7fce75cb50427fd9f9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Mon, 27 Jul 2015 02:35:21 +0200 Subject: [PATCH 028/143] Limit the number of log lines in *-lines.conf actions --- ChangeLog | 2 ++ config/action.d/mail-whois-lines.conf | 6 +++++- config/action.d/sendmail-geoip-lines.conf | 6 +++++- config/action.d/sendmail-whois-lines.conf | 5 ++++- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index de3de7f1..a0e7250d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released - IMPORTANT incompatible changes: * filter.d/roundcube-auth.conf - Changed logpath to 'errors' log (was 'userlogins') + * mail-whois-lines, sendmail-geoip-lines and sendmail-whois-lines + actions include only the first 1000 log lines in the email - Fixes: * reload in interactive mode appends all the jails twice (gh-825) diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 5f760ac8..feca7e99 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -42,7 +42,7 @@ actionban = printf %%b "Hi,\n Here is more information about :\n `whois || echo missing whois program`\n\n Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" @@ -67,3 +67,7 @@ dest = root # Path to the log files which contain relevant lines for the abuser IP # logpath = /dev/null + +# Number of log lines to include in the email +# +grepopts = -m 1000 diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index 3a0c931e..8e311d26 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -34,7 +34,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "" | cut -d':' -f2-` hostname: `host -t A 2>&1`\n\n Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f @@ -47,3 +47,7 @@ name = default # Path to the log files which contain relevant lines for the abuser IP # logpath = /dev/null + +# Number of log lines to include in the email +# +grepopts = -m 1000 diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index abbd60bc..aa00d7bb 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -26,7 +26,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f @@ -40,3 +40,6 @@ name = default # logpath = /dev/null +# Number of log lines to include in the email +# +grepopts = -m 1000 From 7112e4f6c6e8174827e25776303e078add561a4c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 26 Jul 2015 20:41:43 -0400 Subject: [PATCH 029/143] BF: kill the entire process group upon timeout (Close #1129) Requires also establishing a new process group for a child process, which changes previous behavior --- fail2ban/server/action.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 60cb00e4..c58fde2c 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -565,7 +565,9 @@ class CommandAction(ActionBase): stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") try: popen = subprocess.Popen( - realCmd, stdout=stdout, stderr=stderr, shell=True) + realCmd, stdout=stdout, stderr=stderr, shell=True, + preexec_fn=os.setsid # so that killpg does not kill our process + ) stime = time.time() retcode = popen.poll() while time.time() - stime <= timeout and retcode is None: @@ -574,11 +576,12 @@ class CommandAction(ActionBase): if retcode is None: logSys.error("%s -- timed out after %i seconds." % (realCmd, timeout)) - os.kill(popen.pid, signal.SIGTERM) # Terminate the process + pgid = os.getpgid(popen.pid) + os.killpg(pgid, signal.SIGTERM) # Terminate the process time.sleep(0.1) retcode = popen.poll() if retcode is None: # Still going... - os.kill(popen.pid, signal.SIGKILL) # Kill the process + os.killpg(pgid, signal.SIGKILL) # Kill the process time.sleep(0.1) retcode = popen.poll() except OSError, e: From 515ad6dc12f8d29ccfe13767e40eb1bb4c21ac43 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 26 Jul 2015 21:18:36 -0400 Subject: [PATCH 030/143] TST: test to verify killing stuck children processes --- fail2ban/tests/actiontestcase.py | 42 +++++++++++++++++++++++++++++++- fail2ban/tests/utils.py | 27 ++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 5850309e..febbc619 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -24,12 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import os import time +import tempfile from ..server.action import CommandAction, CallingMap from .utils import LogCaptureTestCase - +from .utils import pid_exists class CommandActionTest(LogCaptureTestCase): @@ -202,6 +204,44 @@ class CommandActionTest(LogCaptureTestCase): or self._is_logged('sleep 60 -- timed out after 3 seconds')) self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) + def testExecuteTimeoutWithNastyChildren(self): + # temporary file for a nasty kid shell script + tmpFilename = tempfile.mktemp(".sh", "fail2ban_") + # Create a nasty script which would hang there for a while + with open(tmpFilename, 'w') as f: + f.write("""#!/bin/bash + trap : HUP EXIT TERM + + echo "$$" > %s.pid + echo "my pid $$ . sleeping lo-o-o-ong" + sleep 10000 + """ % tmpFilename) + + def getnastypid(): + with open(tmpFilename + '.pid') as f: + return int(f.read()) + + # First test if can kill the bastard + self.assertRaises( + RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) + # Verify that the proccess itself got killed + self.assertFalse(pid_exists(getnastypid())) # process should have been killed + self.assertTrue(self._is_logged('timed out')) + self.assertTrue(self._is_logged('killed with SIGTERM')) + + # A bit evolved case even though, previous test already tests killing children processes + self.assertRaises( + RuntimeError, CommandAction.executeCmd, 'out=`bash %s`; echo ALRIGHT' % tmpFilename, + timeout=.2) + # Verify that the proccess itself got killed + self.assertFalse(pid_exists(getnastypid())) + self.assertTrue(self._is_logged('timed out')) + self.assertTrue(self._is_logged('killed with SIGTERM')) + + os.unlink(tmpFilename) + os.unlink(tmpFilename + '.pid') + + def testCaptureStdOutErr(self): CommandAction.executeCmd('echo "How now brown cow"') self.assertTrue(self._is_logged("'How now brown cow\\n'")) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 89539107..e7993ead 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -237,3 +237,30 @@ class LogCaptureTestCase(unittest.TestCase): def printLog(self): print(self._log.getvalue()) + +# Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid +# under cc by-sa 3.0 +if os.name == 'posix': + def pid_exists(pid): + """Check whether pid exists in the current process table.""" + import errno + if pid < 0: + return False + try: + os.kill(pid, 0) + except OSError as e: + return e.errno == errno.EPERM + else: + return True +else: + def pid_exists(pid): + import ctypes + kernel32 = ctypes.windll.kernel32 + SYNCHRONIZE = 0x100000 + + process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid) + if process != 0: + kernel32.CloseHandle(process) + return True + else: + return False \ No newline at end of file From 333dd842f971d0e69aae8fcabe9e858965e6c116 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 26 Jul 2015 22:44:52 -0400 Subject: [PATCH 031/143] DOC: moved and adjusted changelog entry from 0.9.2 within 0.9.3 to come --- ChangeLog | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6f83bdea..319c6d18 100644 --- a/ChangeLog +++ b/ChangeLog @@ -50,6 +50,17 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released * filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add HEAD method verb * Revamp of Travis and coverage automated testing + * Character detection heuristics for whois output via optional setting + in mail-whois*.conf. Thanks Thomas Mayer. + Not enabled by default, if _whois_command is set to be + %(_whois_convert_charset)s (e.g. in action.d/mail-whois-common.local), + it + - detects character set of whois output (which is undefined by + RFC 3912) via heuristics of the file command + - converts whois data to UTF-8 character set with iconv + - sends the whois output in UTF-8 character set to mail program + - avoids that heirloom mailx creates binary attachment for input with + unknown character set ver. 0.9.2 (2015/04/29) - better-quick-now-than-later @@ -133,12 +144,6 @@ ver. 0.9.2 (2015/04/29) - better-quick-now-than-later * Absorbed DNSUtils.cidr into addr2bin in filter.py, added unittests * Added syslogsocket configuration to fail2ban.conf * Note in the jail.conf for the recidive jail to increase dbpurgeage (gh-964) - * Character detection heuristics for whois output via optional setting in mail-whois*.conf. Thanks Thomas Mayer - When set by user, - - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command - - converts whois data to UTF-8 character set with iconv - - sends the whois output in UTF-8 character set to mail program - - avoids that heirloom mailx creates binary attachment for input with unknown character set ver. 0.9.1 (2014/10/29) - better, faster, stronger From 0041bc3770a32c736f29413c28683461307491ab Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 26 Jul 2015 23:10:08 -0400 Subject: [PATCH 032/143] DOC: Changelog for shorewall-ipset-proto6.conf + adjusted its description --- ChangeLog | 5 +++- config/action.d/shorewall-ipset-proto6.conf | 31 +++++++++++---------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1b7e1ecd..a7b3470d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,7 +37,10 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released opening a closed port by swapping blocktype and returntype * New filters: - froxlor-auth Thanks Joern Muehlencord - - apache-pass - filter Apache access log for successfull authentication + - apache-pass - filter Apache access log for successful authentication + * New actions: + - shorewall-ipset-proto6 - using proto feature of the Shorewall. Still requires + manual pre-configuration of the shorewall. See the action file for detail. * New jails: - pass2allow-ftp - allows FTP traffic after successful HTTP authentication diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 7fbc21bb..1ebcfb01 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -5,33 +5,35 @@ # This is for ipset protocol 6 (and hopefully later) (ipset v6.14). # for shorewall # -# Use this setting in jail.conf to modify the name and the max time in every jain -# action = shorewall-ipset-proto6[name=SSH, bantime=10000] +# Use this setting in jail.conf to modify use this action instead of a +# default one +# +# banaction = shorewall-ipset-proto6 # # This requires the program ipset which is normally in package called ipset. # -# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 +# kernels, and you need Shorewall >= 4.5.5 to use this action. # # The default Shorewall configuration is with "BLACKLISTNEWONLY=Yes" (see # file /etc/shorewall/shorewall.conf). This means that when Fail2ban adds a # new shorewall rule to ban an IP address, that rule will affect only new -# connections. So if the attempter goes on trying using the same connection +# connections. So if the attacker goes on trying using the same connection # he could even log in. In order to get the same behavior of the iptable # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf # file should me modified with "BLACKLISTNEWONLY=No". # -# The use in IPset in shorewall depends of the version you must have at least 4.5.5< version -# of shorewall # -# Enable shorewall to use a blacklist using iptables creating a file /etc/shorewall/blrules -# and adding "DROP net:+f2b-ssh all" one for every jail details in shorewall documentation blacklist. -# To enable restore you ipset You must set SAVE_IPSETS=Yes in shorewall.conf -# Is importan to read this documentation from shorewall http://shorewall.net/ipsets.html +# Enable shorewall to use a blacklist using iptables creating a file +# /etc/shorewall/blrules and adding "DROP net:+f2b-ssh all" and +# similar lines for every jail. To enable restoring you ipset you +# must set SAVE_IPSETS=Yes in shorewall.conf . You can read more +# about ipsets handling in Shorewall at http://shorewall.net/ipsets.html # -# To force create the ipset in the case that somebody delete the ipset create a file -# /etc/shorewall/initdone and add one line for every ipset (this files are in Perl) -# take care of add 1; at the end of the file -# the example is: +# To force creation of the ipset in the case that somebody deletes the +# ipset create a file /etc/shorewall/initdone and add one line for +# every ipset (this files are in Perl) and add 1 at the end of the file. +# The example: # system("/usr/sbin/ipset -quiet -exist create f2b-ssh hash:ip timeout 600 "); # 1; # @@ -40,7 +42,6 @@ # system("/usr/sbin/ipset -quiet destroy f2b-ssh "); # 1; # This must go to the end of the file if not shorewall compilation fails # -# [Definition] From 17472a8b56de0673adca8efc62e84d1ecb271cbf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 10 May 2015 06:36:29 -0400 Subject: [PATCH 033/143] BF: guarantee order of dbfile to be before dbpurgeage (Closes #1048) --- fail2ban/server/transmitter.py | 1 + fail2ban/tests/clientreadertestcase.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 537cd34d..0d9f0fe4 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -139,6 +139,7 @@ class Transmitter: elif name == "dbpurgeage": db = self.__server.getDatabase() if db is None: + logSys.warning("dbpurgeage setting was not in effect since no db yet") return None else: db.purgeage = command[1] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e3f1699a..25633a19 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -622,6 +622,21 @@ class JailsReaderTest(LogCaptureTestCase): configurator.getOptions() configurator.convertToProtocol() commands = configurator.getConfigStream() + + # verify that dbfile comes before dbpurgeage + def find_set(option): + for i, e in enumerate(commands): + if e[0] == 'set' and e[1] == option: + return i + raise ValueError("Did not find command 'set %s' among commands %s" + % (option, commands)) + + # Set up of logging should come first + self.assertEqual(find_set('logtarget'), 1) + self.assertEqual(find_set('loglevel'), 2) + # then dbfile should be before dbpurgeage + self.assertGreater(find_set('dbpurgeage'), find_set('dbfile')) + # and there is logging information left to be passed into the # server self.assertEqual(sorted(commands), From 8a37a46fbb267215eb13b333ca9577e8cf0c87ab Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Oct 2014 01:01:31 +0200 Subject: [PATCH 034/143] bug fix: option 'dbpurgeage' was never set (always default) by start of fail2ban, because of invalid sorting of options ('dbfile' should be always set before other database options) / closes #1048, closes #1050 --- fail2ban/client/fail2banreader.py | 20 ++++++++------------ fail2ban/server/server.py | 2 +- fail2ban/tests/clientreadertestcase.py | 7 ++++--- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 9b12bba7..709f4b5d 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -53,18 +53,14 @@ class Fail2banReader(ConfigReader): self.__opts = ConfigReader.getOptions(self, "Definition", opts) def convert(self): + # Ensure logtarget/level set first so any db errors are captured + # Also dbfile should be set before all other database options. + # So adding order indices into items, to be stripped after sorting, upon return + order = {"syslogsocket":0, "loglevel":1, "logtarget":2, + "dbfile":50, "dbpurgeage":51} stream = list() for opt in self.__opts: - if opt == "loglevel": - stream.append(["set", "loglevel", self.__opts[opt]]) - elif opt == "logtarget": - stream.append(["set", "logtarget", self.__opts[opt]]) - elif opt == "syslogsocket": - stream.append(["set", "syslogsocket", self.__opts[opt]]) - elif opt == "dbfile": - stream.append(["set", "dbfile", self.__opts[opt]]) - elif opt == "dbpurgeage": - stream.append(["set", "dbpurgeage", self.__opts[opt]]) - # Ensure logtarget/level set first so any db errors are captured - return sorted(stream, reverse=True) + if opt in order: + stream.append((order[opt], ["set", opt, self.__opts[opt]])) + return [opt[1] for opt in sorted(stream)] diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 58857986..6d19544d 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,10 +67,10 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } + self.setSyslogSocket("auto") # Set logging level self.setLogLevel("INFO") self.setLogTarget("STDOUT") - self.setSyslogSocket("auto") def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 25633a19..00128c33 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -632,10 +632,11 @@ class JailsReaderTest(LogCaptureTestCase): % (option, commands)) # Set up of logging should come first - self.assertEqual(find_set('logtarget'), 1) - self.assertEqual(find_set('loglevel'), 2) + self.assertEqual(find_set('syslogsocket'), 0) + self.assertEqual(find_set('loglevel'), 1) + self.assertEqual(find_set('logtarget'), 2) # then dbfile should be before dbpurgeage - self.assertGreater(find_set('dbpurgeage'), find_set('dbfile')) + self.assertTrue(find_set('dbpurgeage') > find_set('dbfile')) # and there is logging information left to be passed into the # server From de6985515760d1aeb037bd66f344e2fc34888db7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Jul 2015 10:35:14 -0400 Subject: [PATCH 035/143] Changelog entries for Serge's fixes --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index a7b3470d..830085d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released communication end (gh-1099) * unbanip always deletes ip from database (independent of bantime, also if currently not banned or persistent) + * guarantee order of dbfile to be before dbpurgeage (gh-1048) + * always set 'dbfile' before other database options (gh-1050) - New Features: * RETURN iptables target is now a variable: From a80820e356216b51645439ac93830b9cee8f554f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Jul 2015 22:34:40 -0400 Subject: [PATCH 036/143] Changelog entry for killpg fix --- ChangeLog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index ba124f92..b1aac1bf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,7 +31,7 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released * filter.d/roundcube-auth.conf - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log - * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override + * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at communication end (gh-1099) @@ -39,6 +39,9 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released currently not banned or persistent) * guarantee order of dbfile to be before dbpurgeage (gh-1048) * always set 'dbfile' before other database options (gh-1050) + * kill the entire process group of the child process upon timeout (gh-1129). + Otherwise could lead to resource exhaustion due to hanging whois + processes. - New Features: * RETURN iptables target is now a variable: From 776322cea3d2fb6642bdf7904a5450f203f3705b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 31 Jul 2015 10:12:14 -0400 Subject: [PATCH 037/143] BF: realpath for /var/run/fail2ban Closes #1142 --- ChangeLog | 2 ++ setup.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c5bcc77b..88748bd0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -45,6 +45,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released * kill the entire process group of the child process upon timeout (gh-1129). Otherwise could lead to resource exhaustion due to hanging whois processes. + * resolve /var/run/fail2ban path in setup.py to help installation + on platforms with /var/run -> /run symlink (gh-1142) - New Features: * RETURN iptables target is now a variable: diff --git a/setup.py b/setup.py index 44f11e62..e3c499d2 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ except ImportError: from distutils.command.build_py import build_py from distutils.command.build_scripts import build_scripts import os -from os.path import isfile, join, isdir +from os.path import isfile, join, isdir, realpath import sys import warnings from glob import glob @@ -81,8 +81,9 @@ else: data_files_extra = [] if os.path.exists('/var/run'): # if we are on the system with /var/run -- we are to use it for having fail2ban/ - # directory there for socket file etc - data_files_extra += [('/var/run/fail2ban', '')] + # directory there for socket file etc. + # realpath is used to possibly resolve /var/run -> /run symlink + data_files_extra += [(realpath('/var/run/fail2ban'), '')] # Get version number, avoiding importing fail2ban. # This is due to tests not functioning for python3 as 2to3 takes place later From 70ba5cb0054f0869930b8cd2dc1bb836653dd289 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 31 Jul 2015 21:32:13 -0400 Subject: [PATCH 038/143] Release changes (too much of manual "labor"! ;)) --- ChangeLog | 8 ++++---- README.md | 6 +++--- RELEASE | 2 +- fail2ban/version.py | 2 +- man/fail2ban-client.1 | 6 +++--- man/fail2ban-regex.1 | 4 ++-- man/fail2ban-server.1 | 6 +++--- man/fail2ban-testcases.1 | 4 ++-- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 88748bd0..3e80e7c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,8 +6,8 @@ Fail2Ban: Changelog =================== -ver. 0.9.3 (2015/XX/XXX) - wanna-be-released ------------ +ver. 0.9.3 (2015/08/01) - lets-all-stay-friends +---------- - IMPORTANT incompatible changes: * filter.d/roundcube-auth.conf @@ -21,7 +21,7 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released with empty value for 'lockingopt' in `[Init]` section. * mail-whois-lines, sendmail-geoip-lines and sendmail-whois-lines actions now include by default only the first 1000 log lines in - the emails. Adjust to augment the behavior. + the emails. Adjust to augment the behavior. - Fixes: * reload in interactive mode appends all the jails twice (gh-825) @@ -53,7 +53,7 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released * New type of operation: pass2allow, use fail2ban for "knocking", opening a closed port by swapping blocktype and returntype * New filters: - - froxlor-auth Thanks Joern Muehlencord + - froxlor-auth - Thanks Joern Muehlencord - apache-pass - filter Apache access log for successful authentication * New actions: - shorewall-ipset-proto6 - using proto feature of the Shorewall. Still requires diff --git a/README.md b/README.md index 66f07034..fe941a63 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.2.dev0 2015/xx/xx + v0.9.3 2015/08/01 ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -37,8 +37,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.2.tar.bz2 - cd fail2ban-0.9.2 + tar xvfj fail2ban-0.9.3.tar.bz2 + cd fail2ban-0.9.3 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index 16880efb..d2a0d552 100644 --- a/RELEASE +++ b/RELEASE @@ -185,7 +185,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.3 (2014/XX/XXX) - wanna-be-released + ver. 0.9.4 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: diff --git a/fail2ban/version.py b/fail2ban/version.py index 7f26206a..4a0f220d 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2015 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.2.dev0" +version = "0.9.3" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index ad68377b..a43123da 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.6. -.TH FAIL2BAN-CLIENT "1" "July 2015" "fail2ban-client v0.9.2.dev" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.1. +.TH FAIL2BAN-CLIENT "1" "July 2015" "fail2ban-client v0.9.3" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.9.2.dev reads log file that contains password failure report +Fail2Ban v0.9.3 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index b21fef8e..c1ae40dc 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.6. -.TH FAIL2BAN-REGEX "1" "July 2015" "fail2ban-regex 0.9.2.dev" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.1. +.TH FAIL2BAN-REGEX "1" "July 2015" "fail2ban-regex 0.9.3" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 3b1044b0..4260e748 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.6. -.TH FAIL2BAN-SERVER "1" "July 2015" "fail2ban-server v0.9.2.dev" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.1. +.TH FAIL2BAN-SERVER "1" "July 2015" "fail2ban-server v0.9.3" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.9.2.dev reads log file that contains password failure report +Fail2Ban v0.9.3 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 7d9ae73b..55eedd50 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.6. -.TH FAIL2BAN-TESTCASES "1" "July 2015" "fail2ban-testcases 0.9.2.dev" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.1. +.TH FAIL2BAN-TESTCASES "1" "July 2015" "fail2ban-testcases 0.9.3" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 9ebf01293b731264a9b046b4cf44655461b2eb23 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 1 Aug 2015 09:17:31 -0400 Subject: [PATCH 039/143] Post release tune ups --- ChangeLog | 9 +++++++++ README.md | 2 +- RELEASE | 12 ++++++------ fail2ban/version.py | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3e80e7c3..b07f876e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,15 @@ Fail2Ban: Changelog =================== +ver. 0.9.4 (2015/XX/XXX) - wanna-be-released +----------- + +- Fixes: + +- New Features: + +- Enhancements: + ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/README.md b/README.md index fe941a63..d9539a32 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.3 2015/08/01 + v0.9.3.dev 2015/XX/XX ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/RELEASE b/RELEASE index d2a0d552..ae6738a1 100644 --- a/RELEASE +++ b/RELEASE @@ -61,24 +61,24 @@ Preparation * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: - tar -C /tmp -jxf dist/fail2ban-0.9.3.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.9.4.tar.bz2 * clean up current direcory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.3/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.4/ * Only differences should be files that you don't want distributed. * Ensure the tests work from the tarball:: - cd /tmp/fail2ban-0.9.3/ && bin/fail2ban-testcases + cd /tmp/fail2ban-0.9.4/ && bin/fail2ban-testcases * Add/finalize the corresponding entry in the ChangeLog * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.3.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.4.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -101,7 +101,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.3 + git tag -s 0.9.4 Pre Release =========== @@ -185,7 +185,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.4 (2014/XX/XXX) - wanna-be-released + ver. 0.9.5 (2015/XX/XXX) - wanna-be-released ----------- - Fixes: diff --git a/fail2ban/version.py b/fail2ban/version.py index 4a0f220d..e9521725 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2015 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.3" +version = "0.9.3.dev" From 7de78f4bb8540e391d67383af69e1c34bad786d4 Mon Sep 17 00:00:00 2001 From: Konstantin Manna Date: Sun, 2 Aug 2015 14:46:07 +0200 Subject: [PATCH 040/143] renamed to correct in protocol --- fail2ban/protocol.py | 2 +- man/fail2ban-client.1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 2cace91f..5d9fdd65 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -89,7 +89,7 @@ protocol = [ ["set unbanip ", "manually Unban in "], ["set maxretry ", "sets the number of failures before banning the host for "], ["set maxlines ", "sets the number of to buffer for regex search for "], -["set addaction [ ]", "adds a new action named for . Optionally for a Python based action, a and can be specified, else will be a Command Action"], +["set addaction [ ]", "adds a new action named for . Optionally for a Python based action, a and can be specified, else will be a Command Action"], ["set delaction ", "removes the action from "], ["", "COMMAND ACTION CONFIGURATION", ""], ["set action actionstart ", "sets the start command of the action for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index a43123da..39ac8bf6 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -229,7 +229,7 @@ sets the number of to buffer for regex search for .TP \fBset addaction [ ]\fR -adds a new action named for +adds a new action named for . Optionally for a Python based action, a and can be specified, From d278fbca306d8bdcc5b3ffe34b1cfc3cd8963f0b Mon Sep 17 00:00:00 2001 From: weberho Date: Wed, 26 Aug 2015 14:48:55 +0200 Subject: [PATCH 041/143] Fixed line suspected to be faulty --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index f545ff13..b6f13840 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -348,7 +348,7 @@ logpath = %(lighttpd_error_log)s [roundcube-auth] port = http,https -logpath = logpath = %(roundcube_errors_log)s +logpath = %(roundcube_errors_log)s [openwebmail] From 60fbf7d7505ff1d20e3550152ea4aa7cac1ffe80 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 26 Aug 2015 09:03:23 -0400 Subject: [PATCH 042/143] changelog for freshly merged PR (roundcube-auth definition of logpath) --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b07f876e..4fe225da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released ----------- - Fixes: + * roundcube-auth jail typo for logpath - New Features: From f7af93a677f8b87477c2c6bc4f1b82095be3780b Mon Sep 17 00:00:00 2001 From: weberho Date: Wed, 26 Aug 2015 15:25:59 +0200 Subject: [PATCH 043/143] Added configuration for opensuse path --- config/paths-opensuse.conf | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 config/paths-opensuse.conf diff --git a/config/paths-opensuse.conf b/config/paths-opensuse.conf new file mode 100644 index 00000000..0d6ad522 --- /dev/null +++ b/config/paths-opensuse.conf @@ -0,0 +1,38 @@ +# openSUSE log-file locations + +[INCLUDES] + +before = paths-common.conf + +after = paths-overrides.local + + +[DEFAULT] + +syslog_local0 = /var/log/messages + +syslog_mail = /var/log/mail + +syslog_mail_warn = %(syslog_mail)s + +syslog_authpriv = %(syslog_local0)s + +syslog_user = %(syslog_local0)s + +syslog_ftp = %(syslog_local0)s + +syslog_daemon = %(syslog_local0)s + +apache_error_log = /var/log/apache2/*error_log + +apache_access_log = /var/log/apache2/*access_log + +pureftpd_log = %(syslog_local0)s + +exim_main_log = /var/log/exim/main.log + +mysql_log = /var/log/mysql/mysqld.log + +roundcube_errors_log = /srv/www/roundcubemail/logs/errors + +solidpop3d_log = %(syslog_mail)s From 2d69fd20aea1dedd74996577f54d3b20b4493eaf Mon Sep 17 00:00:00 2001 From: weberho Date: Wed, 26 Aug 2015 15:37:14 +0200 Subject: [PATCH 044/143] Updated ChangeLog regarding openSUSE's path config --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b07f876e..a8922dec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - New Features: - Enhancements: + * Added openSUSE path configuration ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 835b3ff483b0b62af6c48fdab3aff23730849d48 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sat, 5 Sep 2015 00:12:28 -0400 Subject: [PATCH 045/143] Update apache-badbots.conf Useragent strings including `+http` need to be escaped to be valid. --- config/filter.d/apache-badbots.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index 75c0de03..48b30666 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -8,7 +8,7 @@ [Definition] badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider -badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, +http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 +badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ From 03460d5ed0be3f6b22fd396710ac71f731e1a830 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sun, 6 Sep 2015 01:05:52 -0400 Subject: [PATCH 046/143] Update gen_badbots Added plus char to the regex for escaping user-agent strings. --- files/gen_badbots | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/gen_badbots b/files/gen_badbots index 278058f7..52732317 100755 --- a/files/gen_badbots +++ b/files/gen_badbots @@ -42,7 +42,7 @@ done \ | grep -h -B4 'S '\ | sed -e 's/ //g' \ | awk '/^--/{getline; gsub(" ",""); print $0}' \ -| sed -e 's/\([.\:|()]\)/\\\1/g' \ +| sed -e 's/\([.\:|()+]\)/\\\1/g' \ | uniq \ | tr '\n' '|' \ | sed -e 's/|$//g' From 4bd7991573c266f8d674706b27c816f137676c0d Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sun, 6 Sep 2015 01:12:19 -0400 Subject: [PATCH 047/143] Added apache-badbots.conf --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4fe225da..f27e7dfc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends * filter.d/roundcube-auth.conf - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log + * filter.d/apache-badbots.conf + - Updated useragent string regex * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at From 4cf3b576b9feb0525719aa5d0a614d397bb569ea Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2015 11:14:46 +0200 Subject: [PATCH 048/143] Bugfix for dnsToIp resolver for fqdn with large list of IPs; closes #1164 --- ChangeLog | 1 + fail2ban/server/filter.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f27e7dfc..d49ee82a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Fixes: * roundcube-auth jail typo for logpath + * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) - New Features: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 18afb135..6fc2cd6c 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -855,8 +855,9 @@ class DNSUtils: """ Convert a DNS into an IP address using the Python socket module. Thanks to Kevin Drapel. """ + # retrieve ip (todo: use AF_INET6 for IPv6) try: - return set(socket.gethostbyname_ex(dns)[2]) + return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)]) except socket.error, e: logSys.warning("Unable to find a corresponding IP address for %s: %s" % (dns, e)) From f5b88bd377baca4f08ed93bc7d93468eb1565d69 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Fri, 11 Sep 2015 10:12:57 -0400 Subject: [PATCH 049/143] Updated Changelog --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index d49ee82a..2fec408a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Fixes: * roundcube-auth jail typo for logpath * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) + * filter.d/apache-badbots.conf + - Updated useragent string regex adding escape for `+` - New Features: @@ -45,8 +47,6 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends * filter.d/roundcube-auth.conf - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log - * filter.d/apache-badbots.conf - - Updated useragent string regex * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at From 85b298e49cad821d0fccce92670e618088f0aa4b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 12:56:28 -0400 Subject: [PATCH 050/143] RF: try/except/finally in a single statement (while at it) since we support now python >= 2.6 --- fail2ban/server/action.py | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index c58fde2c..a8e61fb4 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -560,32 +560,32 @@ class CommandAction(ActionBase): return True _cmd_lock.acquire() - try: # Try wrapped within another try needed for python version < 2.5 + try: stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") - try: - popen = subprocess.Popen( - realCmd, stdout=stdout, stderr=stderr, shell=True, - preexec_fn=os.setsid # so that killpg does not kill our process - ) - stime = time.time() + + popen = subprocess.Popen( + realCmd, stdout=stdout, stderr=stderr, shell=True, + preexec_fn=os.setsid # so that killpg does not kill our process + ) + stime = time.time() + retcode = popen.poll() + while time.time() - stime <= timeout and retcode is None: + time.sleep(0.1) retcode = popen.poll() - while time.time() - stime <= timeout and retcode is None: + if retcode is None: + logSys.error("%s -- timed out after %i seconds." % + (realCmd, timeout)) + pgid = os.getpgid(popen.pid) + os.killpg(pgid, signal.SIGTERM) # Terminate the process + time.sleep(0.1) + retcode = popen.poll() + if retcode is None: # Still going... + os.killpg(pgid, signal.SIGKILL) # Kill the process time.sleep(0.1) retcode = popen.poll() - if retcode is None: - logSys.error("%s -- timed out after %i seconds." % - (realCmd, timeout)) - pgid = os.getpgid(popen.pid) - os.killpg(pgid, signal.SIGTERM) # Terminate the process - time.sleep(0.1) - retcode = popen.poll() - if retcode is None: # Still going... - os.killpg(pgid, signal.SIGKILL) # Kill the process - time.sleep(0.1) - retcode = popen.poll() - except OSError, e: - logSys.error("%s -- failed with %s" % (realCmd, e)) + except OSError as e: + logSys.error("%s -- failed with %s" % (realCmd, e)) finally: _cmd_lock.release() From 7cbb3980ebaf123eec0013169172f15c2877e711 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 19:39:44 -0400 Subject: [PATCH 051/143] BF+RF: only return bool status on failed commands execution + mitigate different exit codes between bash/dash Closes #1155 --- fail2ban/server/action.py | 12 +++++++----- fail2ban/tests/actiontestcase.py | 14 ++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index a8e61fb4..de0c8efc 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -561,6 +561,7 @@ class CommandAction(ActionBase): _cmd_lock.acquire() try: + retcode = None # to guarantee being defined upon early except stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") @@ -603,15 +604,16 @@ class CommandAction(ActionBase): return True elif retcode is None: logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid)) - elif retcode < 0: - logSys.error("%s -- killed with %s" % - (realCmd, signame.get(-retcode, "signal %i" % -retcode))) + elif retcode < 0 or retcode > 128: + # dash would return negative while bash 128 + n + sigcode = -retcode if retcode < 0 else retcode - 128 + logSys.error("%s -- killed with %s (return code: %s)" % + (realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)) else: msg = _RETCODE_HINTS.get(retcode, None) logSys.error("%s -- returned %i" % (realCmd, retcode)) if msg: logSys.info("HINT on %i: %s" % (retcode, msg % locals())) - return False - raise RuntimeError("Command execution failed: %s" % realCmd) + return False diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..9e703bbf 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -196,11 +196,10 @@ class CommandActionTest(LogCaptureTestCase): def testExecuteTimeout(self): stime = time.time() # Should take a minute - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) + self.assertFalse(CommandAction.executeCmd('sleep 60', timeout=2)) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') + self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') or self._is_logged('sleep 60 -- timed out after 3 seconds')) self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) @@ -222,17 +221,16 @@ class CommandActionTest(LogCaptureTestCase): return int(f.read()) # First test if can kill the bastard - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) + self.assertFalse(CommandAction.executeCmd( + 'bash %s' % tmpFilename, timeout=.1)) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) # A bit evolved case even though, previous test already tests killing children processes - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'out=`bash %s`; echo ALRIGHT' % tmpFilename, - timeout=.2) + self.assertFalse(CommandAction.executeCmd( + 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2)) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) self.assertTrue(self._is_logged('timed out')) From 5ed731d3b3cc99278c21219de70264b1879251cf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 19:59:55 -0400 Subject: [PATCH 052/143] RF(TST): self.assertTrue(self._is_logged()) -> self.assertLogged and corresponding one for not + support for multiple entries at once, and failure message listing actual log to ease troubleshooting --- fail2ban/tests/actionstestcase.py | 22 ++++++++-------- fail2ban/tests/actiontestcase.py | 36 ++++++++++++++------------ fail2ban/tests/clientreadertestcase.py | 28 ++++++++++---------- fail2ban/tests/databasetestcase.py | 2 +- fail2ban/tests/filtertestcase.py | 19 +++++++------- fail2ban/tests/servertestcase.py | 2 +- fail2ban/tests/utils.py | 36 ++++++++++++++++++++++++++ 7 files changed, 91 insertions(+), 54 deletions(-) diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index bb295967..0ceb35d5 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -94,15 +94,15 @@ class ExecuteActions(LogCaptureTestCase): "Action", os.path.join(TEST_FILES_DIR, "action.d/action.py"), {'opt1': 'value'}) - self.assertTrue(self._is_logged("TestAction initialised")) + self.assertLogged("TestAction initialised") self.__actions.start() time.sleep(3) - self.assertTrue(self._is_logged("TestAction action start")) + self.assertLogged("TestAction action start") self.__actions.stop() self.__actions.join() - self.assertTrue(self._is_logged("TestAction action stop")) + self.assertLogged("TestAction action stop") self.assertRaises(IOError, self.__actions.add, "Action3", "/does/not/exist.py", {}) @@ -136,10 +136,10 @@ class ExecuteActions(LogCaptureTestCase): {}) self.__actions.start() time.sleep(3) - self.assertTrue(self._is_logged("Failed to start")) + self.assertLogged("Failed to start") self.__actions.stop() self.__actions.join() - self.assertTrue(self._is_logged("Failed to stop")) + self.assertLogged("Failed to stop") def testBanActionsAInfo(self): # Action which deletes IP address from aInfo @@ -155,13 +155,13 @@ class ExecuteActions(LogCaptureTestCase): self.__actions._Actions__checkBan() # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key - self.assertFalse(self._is_logged("Failed to execute ban")) - self.assertTrue(self._is_logged("action1 ban deleted aInfo IP")) - self.assertTrue(self._is_logged("action2 ban deleted aInfo IP")) + self.assertNotLogged("Failed to execute ban") + self.assertLogged("action1 ban deleted aInfo IP") + self.assertLogged("action2 ban deleted aInfo IP") self.__actions._Actions__flushBan() # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key - self.assertFalse(self._is_logged("Failed to execute unban")) - self.assertTrue(self._is_logged("action1 unban deleted aInfo IP")) - self.assertTrue(self._is_logged("action2 unban deleted aInfo IP")) + self.assertNotLogged("Failed to execute unban") + self.assertLogged("action1 unban deleted aInfo IP") + self.assertLogged("action2 unban deleted aInfo IP") diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..a4b453a3 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -143,17 +143,17 @@ class CommandActionTest(LogCaptureTestCase): self.__action.actionunban = "true" self.assertEqual(self.__action.actionunban, 'true') - self.assertFalse(self._is_logged('returned')) + self.assertNotLogged('returned') # no action was actually executed yet self.__action.ban({'ip': None}) - self.assertTrue(self._is_logged('Invariant check failed')) - self.assertTrue(self._is_logged('returned successfully')) + self.assertLogged('Invariant check failed') + self.assertLogged('returned successfully') def testExecuteActionEmptyUnban(self): self.__action.actionunban = "" self.__action.unban({}) - self.assertTrue(self._is_logged('Nothing to do')) + self.assertLogged('Nothing to do') def testExecuteActionStartCtags(self): self.__action.HOST = "192.0.2.0" @@ -168,7 +168,7 @@ class CommandActionTest(LogCaptureTestCase): self.__action.actionban = "rm /tmp/fail2ban.test" self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" self.assertRaises(RuntimeError, self.__action.ban, {'ip': None}) - self.assertTrue(self._is_logged('Unable to restore environment')) + self.assertLogged('Unable to restore environment') def testExecuteActionChangeCtags(self): self.assertRaises(AttributeError, getattr, self.__action, "ROST") @@ -187,11 +187,11 @@ class CommandActionTest(LogCaptureTestCase): def testExecuteActionStartEmpty(self): self.__action.actionstart = "" self.__action.start() - self.assertTrue(self._is_logged('Nothing to do')) + self.assertLogged('Nothing to do') def testExecuteIncorrectCmd(self): CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null') - self.assertTrue(self._is_logged('HINT on 127: "Command not found"')) + self.assertLogged('HINT on 127: "Command not found"') def testExecuteTimeout(self): stime = time.time() @@ -200,9 +200,11 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') - or self._is_logged('sleep 60 -- timed out after 3 seconds')) - self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) + self.assertLogged({ + 'sleep 60 -- timed out after 2 seconds', + 'sleep 60 -- timed out after 3 seconds' + }) + self.assertLogged('sleep 60 -- killed with SIGTERM') def testExecuteTimeoutWithNastyChildren(self): # temporary file for a nasty kid shell script @@ -226,8 +228,8 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed - self.assertTrue(self._is_logged('timed out')) - self.assertTrue(self._is_logged('killed with SIGTERM')) + self.assertLogged('timed out') + self.assertLogged('killed with SIGTERM') # A bit evolved case even though, previous test already tests killing children processes self.assertRaises( @@ -235,8 +237,8 @@ class CommandActionTest(LogCaptureTestCase): timeout=.2) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) - self.assertTrue(self._is_logged('timed out')) - self.assertTrue(self._is_logged('killed with SIGTERM')) + self.assertLogged('timed out') + self.assertLogged('killed with SIGTERM') os.unlink(tmpFilename) os.unlink(tmpFilename + '.pid') @@ -244,11 +246,11 @@ class CommandActionTest(LogCaptureTestCase): def testCaptureStdOutErr(self): CommandAction.executeCmd('echo "How now brown cow"') - self.assertTrue(self._is_logged("'How now brown cow\\n'")) + self.assertLogged("'How now brown cow\\n'") CommandAction.executeCmd( 'echo "The rain in Spain stays mainly in the plain" 1>&2') - self.assertTrue(self._is_logged( - "'The rain in Spain stays mainly in the plain\\n'")) + self.assertLogged( + "'The rain in Spain stays mainly in the plain\\n'") def testCallingMap(self): mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'), diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 00128c33..94fe1828 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -173,16 +173,16 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged('No filter set for jail emptyaction')) - self.assertTrue(self._is_logged('No actions were defined for emptyaction')) + self.assertLogged('No filter set for jail emptyaction') + self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged("Found no accessible config files for 'filter.d/catchallthebadies' under %s" % IMPERFECT_CONFIG)) - self.assertTrue(self._is_logged('Unable to read the filter')) + self.assertLogged("Found no accessible config files for 'filter.d/catchallthebadies' under %s" % IMPERFECT_CONFIG) + self.assertLogged('Unable to read the filter') def testJailActionBrokenDef(self): jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, @@ -190,13 +190,13 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged('Error in action definition joho[foo')) + self.assertLogged('Error in action definition joho[foo') # This unittest has been deactivated for some time... - # self.assertTrue(self._is_logged( - # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')) + # self.assertLogged( + # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0') # let's test for what is actually logged and handle changes in the future - self.assertTrue(self._is_logged( - "Caught exception: 'NoneType' object has no attribute 'endswith'")) + self.assertLogged( + "Caught exception: 'NoneType' object has no attribute 'endswith'") if STOCK: def testStockSSHJail(self): @@ -221,7 +221,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(('mail--ho_is', {}), JailReader.extractOptions("mail--ho_is['s']")) #self.printLog() - #self.assertTrue(self._is_logged("Invalid argument ['s'] in ''s''")) + #self.assertLogged("Invalid argument ['s'] in ''s''") self.assertEqual(('mail', {'a': ','}), JailReader.extractOptions("mail[a=',']")) @@ -265,7 +265,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(JailReader._glob(os.path.join(d, '*')), [f1]) # since f2 is dangling -- empty list self.assertEqual(JailReader._glob(f2), []) - self.assertTrue(self._is_logged('File %s is a dangling link, thus cannot be monitored' % f2)) + self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) os.remove(f1) os.remove(f2) @@ -463,8 +463,8 @@ class JailsReaderTest(LogCaptureTestCase): ['start', 'missinglogfiles'], ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'],])) - self.assertTrue(self._is_logged("Errors in jail 'missingbitsjail'. Skipping...")) - self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction")) + self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...") + self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction") if STOCK: def testReadStockActionConf(self): @@ -496,7 +496,7 @@ class JailsReaderTest(LogCaptureTestCase): #old_comm_commands = comm_commands[:] # make a copy #self.assertRaises(ValueError, jails.getOptions, "BOGUS") #self.printLog() - #self.assertTrue(self._is_logged("No section: 'BOGUS'")) + #self.assertLogged("No section: 'BOGUS'") ## and there should be no side-effects #self.assertEqual(jails.convert(), old_comm_commands) diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index dd813ee6..3d156eda 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -316,7 +316,7 @@ class DatabaseTest(LogCaptureTestCase): ticket.setAttempt(5) self.jail.putFailTicket(ticket) actions._Actions__checkBan() - self.assertTrue(self._is_logged("ban ainfo %s, %s, %s, %s" % (True, True, True, True))) + self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) def testPurge(self): if Fail2BanDb is None: # pragma: no cover diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 891b55a6..acce9625 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -260,14 +260,14 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addFailRegex('') self.filter.processLineAndAdd('1387203300.222 192.168.1.32') - self.assertTrue(self._is_logged('Ignore 192.168.1.32')) + self.assertLogged('Ignore 192.168.1.32') tearDownMyTime() def testIgnoreAddBannedIP(self): self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addBannedIP('192.168.1.32') - self.assertFalse(self._is_logged('Ignore 192.168.1.32')) - self.assertTrue(self._is_logged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')) + self.assertNotLogged('Ignore 192.168.1.32') + self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.') def testIgnoreCommand(self): self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py ")) @@ -278,11 +278,11 @@ class IgnoreIP(LogCaptureTestCase): ip = "93.184.216.34" for ignore_source in ["dns", "ip", "command"]: self.filter.logIgnoreIp(ip, True, ignore_source=ignore_source) - self.assertTrue(self._is_logged("[%s] Ignore %s by %s" % (self.jail.name, ip, ignore_source))) + self.assertLogged("[%s] Ignore %s by %s" % (self.jail.name, ip, ignore_source)) def testIgnoreCauseNOK(self): self.filter.logIgnoreIp("example.com", False, ignore_source="NOT_LOGGED") - self.assertFalse(self._is_logged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED"))) + self.assertNotLogged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED")) class IgnoreIPDNS(IgnoreIP): @@ -382,18 +382,17 @@ class LogFileMonitor(LogCaptureTestCase): def testNoLogFile(self): _killfile(self.file, self.name) self.filter.getFailures(self.name) - failure_was_logged = self._is_logged('Unable to open %s' % self.name) - self.assertTrue(failure_was_logged) + self.assertLogged('Unable to open %s' % self.name) def testRemovingFailRegex(self): self.filter.delFailRegex(0) - self.assertFalse(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertNotLogged('Cannot remove regular expression. Index 0 is not valid') self.filter.delFailRegex(0) - self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertLogged('Cannot remove regular expression. Index 0 is not valid') def testRemovingIgnoreRegex(self): self.filter.delIgnoreRegex(0) - self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertLogged('Cannot remove regular expression. Index 0 is not valid') def testNewChangeViaIsModified(self): # it is a brand new one -- so first we think it is modified diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 57453269..6f946cc0 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -934,7 +934,7 @@ class LoggingTests(LogCaptureTestCase): badThread = _BadThread() badThread.start() badThread.join() - self.assertTrue(self._is_logged("Unhandled exception")) + self.assertLogged("Unhandled exception") finally: sys.__excepthook__ = prev_exchook self.assertEqual(len(x), 1) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e7993ead..9d50646a 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -232,6 +232,42 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() + def assertLogged(self, s): + """Assert that a string was logged + + Preferable to assertTrue(self._is_logged(..))) + since provides message with the actual log. + + Parameters + ---------- + s : string or list/set/tuple of strings + Test should succeed if string (or any of the listed) is present in the log + """ + logged = self._log.getvalue() + + s_iter = s if isinstance(s, (tuple, list, set)) else [s] + for s_ in s_iter: + if s_ in logged: + return + raise AssertionError("%r was not found in the log: %r" % (s, logged)) + + def assertNotLogged(self, s): + """Assert that a string was not logged + + Parameters + ---------- + s : string or list/set/tuple of strings + Test should succeed if the string (or at least one of the listed) is not present in the log + """ + logged = self._log.getvalue() + + s_iter = s if isinstance(s, (tuple, list, set)) else [s] + for s_ in s_iter: + if s_ not in logged: + return + raise AssertionError("%r was found present in the log: %r" % (s, logged)) + + def getLog(self): return self._log.getvalue() From 8a4dcafc8f9455720805054bd027af790bf75673 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 21:37:40 -0400 Subject: [PATCH 053/143] RF/BF: py26 has no {} sets, so just pass multiple entries as *args --- fail2ban/tests/actiontestcase.py | 4 ++-- fail2ban/tests/utils.py | 23 ++++++++++------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index a4b453a3..1cba0bcc 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -200,10 +200,10 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertLogged({ + self.assertLogged( 'sleep 60 -- timed out after 2 seconds', 'sleep 60 -- timed out after 3 seconds' - }) + ) self.assertLogged('sleep 60 -- killed with SIGTERM') def testExecuteTimeoutWithNastyChildren(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 9d50646a..35fa59fd 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -232,8 +232,8 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() - def assertLogged(self, s): - """Assert that a string was logged + def assertLogged(self, *s): + """Assert that one of the strings was logged Preferable to assertTrue(self._is_logged(..))) since provides message with the actual log. @@ -244,28 +244,25 @@ class LogCaptureTestCase(unittest.TestCase): Test should succeed if string (or any of the listed) is present in the log """ logged = self._log.getvalue() - - s_iter = s if isinstance(s, (tuple, list, set)) else [s] - for s_ in s_iter: + for s_ in s: if s_ in logged: return - raise AssertionError("%r was not found in the log: %r" % (s, logged)) + raise AssertionError("None among %r was found in the log: %r" % (s, logged)) - def assertNotLogged(self, s): - """Assert that a string was not logged + def assertNotLogged(self, *s): + """Assert that strings were not logged Parameters ---------- s : string or list/set/tuple of strings - Test should succeed if the string (or at least one of the listed) is not present in the log + Test should succeed if the string (or at least one of the listed) is not + present in the log """ logged = self._log.getvalue() - - s_iter = s if isinstance(s, (tuple, list, set)) else [s] - for s_ in s_iter: + for s_ in s: if s_ not in logged: return - raise AssertionError("%r was found present in the log: %r" % (s, logged)) + raise AssertionError("All of the %r were found present in the log: %r" % (s, logged)) def getLog(self): From 63c7ceb81d2a3e46c916a2e3c530fa7ec83f8c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 10:57:51 +0300 Subject: [PATCH 054/143] logrotate: Remove outdated Fedora comment --- files/fail2ban-logrotate | 3 --- 1 file changed, 3 deletions(-) diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index a09870af..ef4da852 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -4,9 +4,6 @@ # # Debian: # https://github.com/fail2ban/fail2ban/blob/debian/debian/fail2ban.logrotate -# -# Fedora view: -# http://pkgs.fedoraproject.org/cgit/fail2ban.git/tree/fail2ban-logrotate /var/log/fail2ban.log { rotate 7 From 67a94733a9b3eec7e325dcdbbf411473770d36bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 11:01:22 +0300 Subject: [PATCH 055/143] logrotate: Do not rotate empty logs As a useful side effect, prevents "Unable to contact server. Is it running?" mails from cron when fail2ban hasn't been (intentionally) running nor thus logging anything either. --- ChangeLog | 1 + files/fail2ban-logrotate | 1 + 2 files changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2fec408a..811473e7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - New Features: - Enhancements: + * Do not rotate empty log files ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index ef4da852..8d94a8b3 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -8,6 +8,7 @@ /var/log/fail2ban.log { rotate 7 missingok + notifempty compress postrotate /usr/bin/fail2ban-client flushlogs 1>/dev/null || true From 943efdb1a073668096613d6abc4ade4985928b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 11:08:04 +0300 Subject: [PATCH 056/143] Comment spelling fixes --- fail2ban/tests/actiontestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..0749edc7 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -224,7 +224,7 @@ class CommandActionTest(LogCaptureTestCase): # First test if can kill the bastard self.assertRaises( RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) - # Verify that the proccess itself got killed + # Verify that the process itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) @@ -233,7 +233,7 @@ class CommandActionTest(LogCaptureTestCase): self.assertRaises( RuntimeError, CommandAction.executeCmd, 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2) - # Verify that the proccess itself got killed + # Verify that the process itself got killed self.assertFalse(pid_exists(getnastypid())) self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) From fbdd0b74a10f65336724e78ac9b03b8eb86b84d6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Sep 2015 10:45:39 -0400 Subject: [PATCH 057/143] DOC: Changelog entry for this fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 811473e7..4a95727b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * Treat failed and killed execution of commands identically (only + different log messages), which addresses different behavior on different + exit codes of dash and bash (gh-1155) - New Features: From db1a3f17e18781d5422daf58e45bb3a811b19b47 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 16 Sep 2015 08:56:46 -0400 Subject: [PATCH 058/143] ENH: new date pattern with year after day (not after entire entry) --- ChangeLog | 2 ++ fail2ban/server/datedetector.py | 4 ++++ fail2ban/tests/datedetectortestcase.py | 1 + 3 files changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4a95727b..cf1fcbf4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Enhancements: * Do not rotate empty log files + * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) + http://bugs.debian.org/798923 ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 95d368b5..a4839f5c 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -78,6 +78,10 @@ class DateDetector(object): # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # asctime with optional day, subsecond and/or year coming after day + # http://bugs.debian.org/798923 + # Sun Jan 23 2005 21:59:59.011 + self.appendTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") # simple date, optional subsecond (proftpd): # 2005-01-23 21:59:59 # simple date: 2005/01/23 21:59:59 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 0d758640..facfda6d 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -74,6 +74,7 @@ class DateDetectorTest(unittest.TestCase): (False, "Jan 23 21:59:59"), (False, "Sun Jan 23 21:59:59 2005"), (False, "Sun Jan 23 21:59:59"), + (False, "Sun Jan 23 2005 21:59:59"), (False, "2005/01/23 21:59:59"), (False, "2005.01.23 21:59:59"), (False, "23/01/2005 21:59:59"), From 17a42897988f154f8f52c9babbb924fcadca709c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 08:38:51 -0400 Subject: [PATCH 059/143] BF: relax 1 sec delay testing to 100ms margin (Closes #1195) --- fail2ban/tests/servertestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 6f946cc0..afa6162e 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -169,7 +169,8 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) t1 = time.time() # Approx 1 second delay - self.assertAlmostEqual(t1 - t0, 1, places=1) + self.assertGreater(t1 - t0, 0.99) # shouldn't be faster than 1sec + self.assertGreater(1.1, t1 - t0) def testDatabase(self): tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") From 84afcd8b1f57c3115e77f9b75ae86ba9de98b21f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 09:45:51 -0400 Subject: [PATCH 060/143] BF(PY26): no assertGreater in 2.6 -- use explicit comparison --- fail2ban/tests/servertestcase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index afa6162e..86ffdb46 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -168,9 +168,9 @@ class Transmitter(TransmitterBase): t0 = time.time() self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) t1 = time.time() - # Approx 1 second delay - self.assertGreater(t1 - t0, 0.99) # shouldn't be faster than 1sec - self.assertGreater(1.1, t1 - t0) + # Approx 1 second delay but not faster + dt = t1 - t0 + self.assertTrue(0.99 < dt < 1.1, msg="Sleep was %g sec" % dt) def testDatabase(self): tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") From 8cf614e2219c999564ce669c5ce0dc29364acb29 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 12:13:52 -0400 Subject: [PATCH 061/143] ENH: allow to split ignoreip by space and/or comma (Closes #1197) Way too many people ran into this gotcha, so lets just do it --- ChangeLog | 1 + config/jail.conf | 2 +- fail2ban/client/jailreader.py | 7 +++---- fail2ban/helpers.py | 10 ++++++++++ fail2ban/tests/misctestcase.py | 9 +++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index fea070ba..e2dbf69d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) http://bugs.debian.org/798923 * Added openSUSE path configuration (Thanks Johannes Weberhofer) + * Allow to split ignoreip entries by ',' as well as by ' ' (gh-1197) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/jail.conf b/config/jail.conf index b6f13840..7500f4ff 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -46,7 +46,7 @@ before = paths-debian.conf # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not # ban a host which matches an address in this list. Several addresses can be -# defined using space separator. +# defined using space (and/or comma) separator. ignoreip = 127.0.0.1/8 # External command that will take an tagged arguments to ignore, e.g. , diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6d0fddfa..54ac59fa 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -33,6 +33,7 @@ from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger +from ..helpers import splitcommaspace # Gets the instance of the logger. logSys = getLogger(__name__) @@ -208,10 +209,8 @@ class JailReader(ConfigReader): elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) elif opt == "ignoreip": - for ip in self.__opts[opt].split(): - # Do not send a command if the rule is empty. - if ip != '': - stream.append(["set", self.__name, "addignoreip", ip]) + for ip in splitcommaspace(self.__opts[opt]): + stream.append(["set", self.__name, "addignoreip", ip]) elif opt == "findtime": stream.append(["set", self.__name, "findtime", self.__opts[opt]]) elif opt == "bantime": diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f5c3163a..8e1b0e32 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -127,3 +127,13 @@ def excepthook(exctype, value, traceback): getLogger("fail2ban").critical( "Unhandled exception in Fail2Ban:", exc_info=True) return sys.__excepthook__(exctype, value, traceback) + +def splitcommaspace(s): + """Helper to split on any comma or space + + Returns empty list if input is empty (or None) and filters + out empty entries + """ + if not s: + return [] + return filter(bool, re.split('[ ,]', s)) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c95efa43..e28ce422 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -33,6 +33,7 @@ from glob import glob from StringIO import StringIO from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger +from ..helpers import splitcommaspace from ..server.datetemplate import DatePatternRegex @@ -55,6 +56,14 @@ class HelpersTest(unittest.TestCase): # might be fragile due to ' vs " self.assertEqual(args, "('Very bad', None)") + def testsplitcommaspace(self): + self.assertEqual(splitcommaspace(None), []) + self.assertEqual(splitcommaspace(''), []) + self.assertEqual(splitcommaspace(' '), []) + self.assertEqual(splitcommaspace('1'), ['1']) + self.assertEqual(splitcommaspace(' 1 2 '), ['1', '2']) + self.assertEqual(splitcommaspace(' 1, 2 , '), ['1', '2']) + class SetupTest(unittest.TestCase): From 4744e165394c696144499d95d79ab5b55b556cbd Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Thu, 24 Sep 2015 06:37:01 -0700 Subject: [PATCH 062/143] README :: Some style/grammar tweaks, and init/service script mention. Re: #1193 --- README.md | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d9539a32..a64bb943 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,16 @@ ## Fail2Ban: ban hosts that cause multiple authentication errors -Fail2Ban scans log files like /var/log/pwdfail and bans IP that makes too many -password failures. It updates firewall rules to reject the IP address. These -rules can be defined by the user. Fail2Ban can read multiple log files such as -sshd or Apache web server ones. +Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses +having too many failed login attempts. +It does this by updating system firewall rules to reject new connections +from those IP addresses, for a configurable amount of time. +Fail2Ban comes out-of-the-box ready to read many standard log files, such as those +for sshd and Apache, and is easy to configure to read any log file you choose, for +any error you choose. -Fail2Ban is able to reduce the rate of incorrect authentications attempts -however it cannot eliminate the risk that weak authentication presents. +Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, +it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. @@ -42,7 +45,7 @@ To install, just do: python setup.py install This will install Fail2Ban into the python library directory. The executable -scripts are placed into /usr/bin, and configuration under /etc/fail2ban. +scripts are placed into `/usr/bin`, and configuration under `/etc/fail2ban`. Fail2Ban should be correctly installed now. Just type: @@ -51,11 +54,19 @@ Fail2Ban should be correctly installed now. Just type: to see if everything is alright. You should always use fail2ban-client and never call fail2ban-server directly. +Please note that the system init/service script is not automatically installed. +To enable fail2ban as an automatic service, simply copy the script for your +distro from the `files` directory to `/etc/init.d`. Example: + + cp files/debian-initd /etc/init.d/fail2ban + update-rc.d fail2ban defaults + service fail2ban start + Configuration: -------------- -You can configure Fail2Ban using the files in /etc/fail2ban. It is possible to -configure the server using commands sent to it by fail2ban-client. The +You can configure Fail2Ban using the files in `/etc/fail2ban`. It is possible to +configure the server using commands sent to it by `fail2ban-client`. The available commands are described in the fail2ban-client(1) manpage. Also see fail2ban(1) and jail.conf(5) manpages for further references. From d618ee3d90faeaacea1096ac43af36450527d604 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 24 Sep 2015 09:53:55 -0400 Subject: [PATCH 063/143] BF: disable testing on python 3.2 until coverage gets a fix --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f65e4896..adb41e7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ python: - 2.6 - 2.7 - pypy - - 3.2 + # disabled until coverage module fixes up compatibility issue + # - 3.2 - 3.3 - 3.4 - pypy3 From c1b80a5e1bb9d426e87d4eec3285cf20c405438b Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Fri, 25 Sep 2015 02:23:08 -0700 Subject: [PATCH 064/143] README :: fitted paragraph style --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a64bb943..0be4920a 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,15 @@ ## Fail2Ban: ban hosts that cause multiple authentication errors -Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses -having too many failed login attempts. -It does this by updating system firewall rules to reject new connections -from those IP addresses, for a configurable amount of time. -Fail2Ban comes out-of-the-box ready to read many standard log files, such as those -for sshd and Apache, and is easy to configure to read any log file you choose, for -any error you choose. +Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses having +too many failed login attempts. It does this by updating system firewall rules +to reject new connections from those IP addresses, for a configurable amount +of time. Fail2Ban comes out-of-the-box ready to read many standard log files, +such as those for sshd and Apache, and is easy to configure to read any log +file you choose, for any error you choose. -Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, -it cannot eliminate the risk that weak authentication presents. +Though Fail2Ban is able to reduce the rate of incorrect authentications +attempts, it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. From 0610791ffecaaade6ed844ad59f99b284e203807 Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Fri, 25 Sep 2015 02:25:11 -0700 Subject: [PATCH 065/143] README :: init/service example mentions debian based systems as the example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0be4920a..cbc075e2 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ never call fail2ban-server directly. Please note that the system init/service script is not automatically installed. To enable fail2ban as an automatic service, simply copy the script for your -distro from the `files` directory to `/etc/init.d`. Example: +distro from the `files` directory to `/etc/init.d`. Example (on a Debian-based +system): cp files/debian-initd /etc/init.d/fail2ban update-rc.d fail2ban defaults From 2895d981fa8b91cc6ea2bb74f325453c48dfb31d Mon Sep 17 00:00:00 2001 From: "M. Maraun" Date: Fri, 3 Jul 2015 22:42:22 +0200 Subject: [PATCH 066/143] Set Timeout at urlopen to 3 seconds --- THANKS | 1 + config/action.d/badips.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 7bf723c5..68c7af48 100644 --- a/THANKS +++ b/THANKS @@ -71,6 +71,7 @@ kojiro Lars Kneschke Lee Clemens leftyfb (Mike Rushton) +M. Maraun Manuel Arostegui Ramirez Marcel Dopita Mark Edgington diff --git a/config/action.d/badips.py b/config/action.d/badips.py index a1df00a3..99e1866a 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -117,7 +117,7 @@ class BadIPsAction(ActionBase): """ try: response = urlopen( - self._Request("/".join([self._badips, "get", "categories"]))) + self._Request("/".join([self._badips, "get", "categories"])), None, 3) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) self._logSys.error( From 6c0f898ec7c5165ca50ee683e8cf371ed9b5a5a4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 27 Sep 2015 00:49:57 -0400 Subject: [PATCH 067/143] DOC: changelog for the timeout change --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index fea070ba..c5fd30aa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) http://bugs.debian.org/798923 * Added openSUSE path configuration (Thanks Johannes Weberhofer) + * Added a timeout (3 sec) to urlopen within badips.py action + (Thanks M. Maraun) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 0d8968daa99b4a178271ab1db9de407124a7f26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Wed, 30 Sep 2015 16:07:45 +0200 Subject: [PATCH 068/143] Added CloudFlare API error codes URL --- config/action.d/cloudflare.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4bc90c97..aa87163c 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -9,6 +9,8 @@ # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account +# +# CloudFlare API error codes: https://www.cloudflare.com/docs/host-api.html#s4.2 [Definition] From 36919d9f976f6140a5bc565cb5aa10068b301a2e Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Fri, 2 Oct 2015 15:19:42 -0700 Subject: [PATCH 069/143] ssh.conf: Fix disconnect "Auth fail" matching The regex for matching against "Auth fail" disconnect log message does not match against current versions of ssh. OpenSSH 5.9 introduced privilege separation of the pre-auth process, which included [logging through monitor.c](http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/monitor.c.diff?r1=1.113&r2=1.114) which adds " [preauth]" to the end of each message and causes the log level to be prepended to each message. It also fails to match against clients which send a disconnect message with a description that is either empty or includes a space, since this is the content in the log message after the disconnect code, per [packet.c:1785](http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/packet.c?annotate=1.215), which was matched by \S+. Although I have not observed this yet, I couldn't find anything which would preclude it in [RFC 4253](https://tools.ietf.org/html/rfc4253#section-11.1) and since the message is attacker-controlled it provides a way to avoid getting banned. This commit fixes both issues. Signed-off-by: Kevin Locke --- config/filter.d/sshd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index b000cd49..5fad2b32 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -27,7 +27,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ ^%(__prefix_line)srefused connect from \S+ \(\)\s*$ - ^%(__prefix_line)sReceived disconnect from : 3: \S+: Auth fail$ + ^%(__prefix_line)s(?:error: )?Received disconnect from : 3: .*: Auth fail(?: \[preauth\])?$ ^%(__prefix_line)sUser .+ from not allowed because a group is listed in DenyGroups\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ From 42b0e9258dc90556f63095abedd2ca72dd3f5670 Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Fri, 2 Oct 2015 15:56:26 -0700 Subject: [PATCH 070/143] Test cases for ssh.conf disconnect "Auth fail" Add test coverage for the new disconnect "Auth fail" matching added in 36919d9f. Signed-off-by: Kevin Locke --- fail2ban/tests/files/logs/sshd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 2f4400bb..62204339 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -132,6 +132,12 @@ Nov 23 21:50:37 sshd[7148]: Connection closed by 61.0.0.1 [preauth] # failJSON: { "time": "2005-07-13T18:44:28", "match": true , "host": "89.24.13.192", "desc": "from gh-289" } Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail +# failJSON: { "time": "2004-10-01T17:27:44", "match": true , "host": "94.249.236.6", "desc": "newer format per commit 36919d9f" } +Oct 1 17:27:44 localhost sshd[24077]: error: Received disconnect from 94.249.236.6: 3: com.jcraft.jsch.JSchException: Auth fail [preauth] + +# failJSON: { "time": "2004-10-01T17:27:44", "match": true , "host": "94.249.236.6", "desc": "space in disconnect description per commit 36919d9f" } +Oct 1 17:27:44 localhost sshd[24077]: error: Received disconnect from 94.249.236.6: 3: Ha ha, suckers!: Auth fail [preauth] + # failJSON: { "match": false } Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } From 2a5c93cfb54b1f307d0360c550a7a9477e7cfb0e Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Mon, 5 Oct 2015 00:31:13 -0700 Subject: [PATCH 071/143] Update ChangeLog and THANKS for "Auth fail" changes Document the changes from 36919d9f in the ChangeLog and add myself to the THANKS file (at @sebres suggestion). Signed-off-by: Kevin Locke --- ChangeLog | 2 ++ THANKS | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2c2636a8..3bdd867a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * filter.d/sshd.conf + - Updated "Auth fail" regex for OpenSSH 5.9 and later * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) diff --git a/THANKS b/THANKS index 68c7af48..5cea437b 100644 --- a/THANKS +++ b/THANKS @@ -65,6 +65,7 @@ Joël Bertrand JP Espinosa jserrachinha Justin Shore +Kevin Locke Kévin Drapel kjohnsonecl kojiro From 61ac48170308da590d317cac7e0b846461a86352 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 7 Oct 2015 13:27:45 +0200 Subject: [PATCH 072/143] IpToName test case fixed ('66.249.66.1' resp. 'crawl-66-249-66-1.googlebot.com' seems to be unresolvable) --- fail2ban/tests/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index acce9625..acf15528 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1025,8 +1025,8 @@ class DNSUtilsTests(unittest.TestCase): self.assertEqual(res, []) def testIpToName(self): - res = DNSUtils.ipToName('66.249.66.1') - self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') + res = DNSUtils.ipToName('8.8.4.4') + self.assertEqual(res, 'google-public-dns-b.google.com') # invalid ip (TEST-NET-1 according to RFC 5737) res = DNSUtils.ipToName('192.0.2.0') self.assertEqual(res, None) From 2696ede2514261c67123b1e3ba74b6f73b628369 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 7 Oct 2015 14:34:13 +0200 Subject: [PATCH 073/143] mysqld-auth: Updated "Access denied ..." regex for MySQL 5.6 and later closes gh-1211 --- ChangeLog | 2 ++ config/filter.d/mysqld-auth.conf | 2 +- fail2ban/tests/files/logs/mysqld-auth | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3bdd867a..9e145b65 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * filter.d/mysqld-auth.conf + - Updated "Access denied ..." regex for MySQL 5.6 and later (gh-1211) * filter.d/sshd.conf - Updated "Auth fail" regex for OpenSSH 5.9 and later * Treat failed and killed execution of commands identically (only diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 92dc9a99..f29beffc 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[Warning\] Access denied for user '\w+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[Warning\] Access denied for user '\w+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index aa684266..b75ecfc4 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -15,3 +15,5 @@ Sep 16 21:30:26 catinthehat mysqld: 130916 21:30:26 [Warning] Access denied for # failJSON: { "time": "2004-09-16T21:30:32", "match": true , "host": "74.207.241.159" } Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for user 'hacker'@'74.207.241.159' (using password: NO) +# failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" } +2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES) \ No newline at end of file From 078e2048f237fc0929483f78d620c95cad2b5f33 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Thu, 8 Oct 2015 16:09:46 +0200 Subject: [PATCH 074/143] files: Strip trailing whitespace from files Run the command `StripWhitespace` from the [Vim Better Whitespace Plugin](https://github.com/ntpeters/vim-better-whitespace). --- files/bash-completion | 2 +- files/gen_badbots | 2 +- files/nagios/README | 10 +++++----- files/nagios/check_fail2ban | 22 +++++++++++----------- files/solaris-svc-fail2ban | 2 +- files/suse-initd | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/files/bash-completion b/files/bash-completion index 57ec15b3..36e0cbba 100644 --- a/files/bash-completion +++ b/files/bash-completion @@ -31,7 +31,7 @@ __fail2ban_jail_action_methods () { _fail2ban () { local cur prev words cword - _init_completion || return + _init_completion || return case $prev in -V|--version|-h|--help) diff --git a/files/gen_badbots b/files/gen_badbots index 52732317..75a6a0d5 100755 --- a/files/gen_badbots +++ b/files/gen_badbots @@ -26,7 +26,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the +# along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA. # diff --git a/files/nagios/README b/files/nagios/README index 2b855d83..324260ff 100644 --- a/files/nagios/README +++ b/files/nagios/README @@ -8,8 +8,8 @@ How to use ---------- Just have to run the following command: $ ./check_fail2ban --help - -If you need to use this script with NRPE you just have to do the + +If you need to use this script with NRPE you just have to do the following steps: 1 allow your user to run the script with the sudo rights. Just add @@ -20,7 +20,7 @@ following steps: command[check_fail2ban]=/usr/bin/sudo //check_fail2ban 3 don't forget to restart your NRPE daemon - + /!\ be careful to let no one able to update the check_fail2ban ;) ------------------------------------------------------------------------------ @@ -37,7 +37,7 @@ HELP: 2.) delete the socket if available rm /var/run/fail2ban/fail2ban.sock -3.) start the Service +3.) start the Service /etc/init.d/fail2ban start 4.) check if fail2ban is working @@ -58,7 +58,7 @@ Options: -V, --version Print version information -D, --display=STRING - To modify the output display + To modify the output display default is "CHECK FAIL2BAN ACTIVITY" -P, --path-fail2ban_client=STRING Specify the path to the tw_cli binary diff --git a/files/nagios/check_fail2ban b/files/nagios/check_fail2ban index fc88e654..bbfa63aa 100755 --- a/files/nagios/check_fail2ban +++ b/files/nagios/check_fail2ban @@ -4,12 +4,12 @@ # -=- -=- # ------------------------------------------------------- # -# Description : This plugin checks if the fail2ban server is running +# Description : This plugin checks if the fail2ban server is running # and how many IPs are currently banned. -# +# # # inspired by the work of Sebastian Mueller - http://www.elchtest.eu -# +# # # Version : 0.1 # ------------------------------------------------------- @@ -17,7 +17,7 @@ # - see the How to use section # # Out : -# - only print on the standard output +# - only print on the standard output # # Features : # - perfdata output @@ -51,8 +51,8 @@ # # Just have to run the following command: # $ ./check_fail2ban --help -# -# If you need to use this script with NRPE you just have to do the +# +# If you need to use this script with NRPE you just have to do the # following steps: # # 1 allow your user to run the script with the sudo rights. Just add @@ -64,7 +64,7 @@ # # 3 don't forget to restart your NRPE daemon # -# +# # /!\ be careful to let no one able to update the check_fail2ban ;) # ------------------------------------------------------------------------------ # @@ -251,7 +251,7 @@ Options: -V, --version Print version information -D, --display=STRING - To modify the output display + To modify the output display default is "CHECK FAIL2BAN ACTIVITY" -P, --path-fail2ban_client=STRING Specify the path to the tw_cli binary @@ -269,7 +269,7 @@ Options: If you want to activate the perfdata output -v, --verbose Show details for command-line debugging (Nagios may truncate the output) - + Send email to $a_mail if you have questions regarding use of this software. To submit patches or suggest improvements, send email to $a_mail @@ -315,7 +315,7 @@ sub obtain_jail_list { if ($return_code) { return -1; } - + my @jail_list; foreach (@command_output) { if ($_=~/^.*Jail list:\t+(.*)/) { @@ -323,7 +323,7 @@ sub obtain_jail_list { @jail_list = split(/,/, $1); } } - + return @jail_list; } diff --git a/files/solaris-svc-fail2ban b/files/solaris-svc-fail2ban index e397474b..96415323 100755 --- a/files/solaris-svc-fail2ban +++ b/files/solaris-svc-fail2ban @@ -2,7 +2,7 @@ # # fail2ban This init.d script is used to start fail2ban. # (C) by Hanno Wagner , License is GPL - + #set -x . /lib/svc/share/smf_include.sh diff --git a/files/suse-initd b/files/suse-initd index 09c25687..ddd26ec5 100644 --- a/files/suse-initd +++ b/files/suse-initd @@ -60,12 +60,12 @@ case "$1" in if [ -f $FAIL2BAN_SOCKET ] then - echo "$FAIL2BAN_SOCKET not removed .. removing .." + echo "$FAIL2BAN_SOCKET not removed .. removing .." rm $FAIL2BAN_SOCKET fi if [ -f $FAIL2BAN_PID ] then - echo "$FAIL2BAN_PID not removed .. removing .." + echo "$FAIL2BAN_PID not removed .. removing .." rm $FAIL2BAN_PID fi From 6fb5e3a494d8ec2385e9c44adab7ae3c644f099c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Oct 2015 14:10:02 -0400 Subject: [PATCH 075/143] removed outdated and "problematic" .pydevproject --- .pydevproject | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .pydevproject diff --git a/.pydevproject b/.pydevproject deleted file mode 100644 index 3c3130d9..00000000 --- a/.pydevproject +++ /dev/null @@ -1,12 +0,0 @@ - - - - -python 2.3 - -/fail2ban-0.8/client -/fail2ban-0.8/server -/fail2ban-0.8/testcases -/fail2ban-0.8 - - From 617302fcc2d0b6dee89593ba6d15b58887bdc832 Mon Sep 17 00:00:00 2001 From: agentmoller001 Date: Fri, 9 Oct 2015 18:16:36 -0700 Subject: [PATCH 076/143] Updated route.conf to clear warnings Does not throw warnings when starting/restarting by adding three lines of code. --- config/action.d/route.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/action.d/route.conf b/config/action.d/route.conf index 123245e5..9b96a7b5 100644 --- a/config/action.d/route.conf +++ b/config/action.d/route.conf @@ -17,6 +17,9 @@ [Definition] actionban = ip route add actionunban = ip route del +actioncheck = +actionstart = +actionstop = [Init] From a28e6b442e50bf17cefc886b3873ebd9a418a235 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Tue, 13 Oct 2015 09:55:28 +0200 Subject: [PATCH 077/143] Add check in apache-fakegooglebot to protect against PTR fake record An attacker may return a PTR record which fakes a Googlebot's domain name. This modification resolves the PTR records to verify it. See "Verifying Googlebot": --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/ignorecommands/apache-fakegooglebot | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 9e145b65..2f5a158b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Allow to split ignoreip entries by ',' as well as by ' ' (gh-1197) * Added a timeout (3 sec) to urlopen within badips.py action (Thanks M. Maraun) + * Added check against atacker's Googlebot PTR fake records + (Thanks Pablo Rodriguez Fernandez) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/THANKS b/THANKS index 5cea437b..45674847 100644 --- a/THANKS +++ b/THANKS @@ -89,6 +89,7 @@ Mika (mkl) Nick Munger onorua Orion Poplawski +Pablo Rodriguez Fernandez Paul Marrapese Paul Traina Noel Butler diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 47ef51f6..3028d86a 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,10 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - sys.exit(0 if (host and re.match('crawl-.*\.googlebot\.com', host)) else 1) + if not host or not re.match('crawl-.*\.googlebot\.com', host): + sys.exit(1) + host_ips = DNSUtils.dnsToIp(host) + sys.exit(0 if ip in host_ips else 1) if __name__ == '__main__': is_googlebot(process_args(sys.argv)) From 7e6964dd9d0a21cc2219591c628bfd68e8ec0726 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 15 Oct 2015 10:40:56 +0200 Subject: [PATCH 078/143] Fix section jail.conf.5 manpage The section of jail.conf manpage is wrong, should be 5, not 10 --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 45eea040..957a04b4 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -1,4 +1,4 @@ -.TH JAIL.CONF "10" "October 2013" "Fail2Ban" "Fail2Ban Configuration" +.TH JAIL.CONF "5" "October 2013" "Fail2Ban" "Fail2Ban Configuration" .SH NAME jail.conf \- configuration for the fail2ban server .SH SYNOPSIS From 75d33c0f096bda44de2f4bb4cd41703a3e4b86d7 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 7 Oct 2015 19:50:03 -0600 Subject: [PATCH 079/143] Add *_backend options for services to allow distros to set the default backend per service. Set default to systemd for Fedora as appropriate. --- ChangeLog | 2 ++ config/jail.conf | 28 +++++++++++++++++++++++++++- config/paths-common.conf | 10 ++++++++++ config/paths-fedora.conf | 12 ++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2f5a158b..08708a08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks M. Maraun) * Added check against atacker's Googlebot PTR fake records (Thanks Pablo Rodriguez Fernandez) + * Add *_backend options for services to allow distros to set the default + backend per service, set default to systemd for Fedora as appropriate ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/jail.conf b/config/jail.conf index 7500f4ff..36fe6bcf 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -216,6 +216,7 @@ action = %(action_)s port = ssh logpath = %(sshd_log)s +backend = %(sshd_backend)s [sshd-ddos] @@ -224,12 +225,14 @@ logpath = %(sshd_log)s # in the body. port = ssh logpath = %(sshd_log)s +backend = %(sshd_backend)s [dropbear] port = ssh logpath = %(dropbear_log)s +backend = %(dropbear_backend)s [selinux-ssh] @@ -393,6 +396,7 @@ maxretry = 5 port = http,https logpath = %(syslog_daemon)s +backend = %(syslog_backend)s [guacamole] @@ -410,12 +414,14 @@ logpath = /var/log/monit port = 10000 logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s [froxlor-auth] port = http,https logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s # @@ -444,12 +450,14 @@ logpath = /var/log/3proxy.log port = ftp,ftp-data,ftps,ftps-data logpath = %(proftpd_log)s +backend = %(proftpd_backend)s [pure-ftpd] port = ftp,ftp-data,ftps,ftps-data logpath = %(pureftpd_log)s +backend = %(pureftpd_backend)s maxretry = 6 @@ -457,6 +465,7 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(syslog_daemon)s +backend = %(syslog_backend)s maxretry = 6 @@ -464,6 +473,7 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(wuftpd_log)s +backend = %(wuftpd_backend)s maxretry = 6 @@ -491,18 +501,21 @@ logpath = /root/path/to/assp/logs/maillog.txt port = smtp,465,submission logpath = %(syslog_mail)s +backend = %(syslog_backend)s [postfix] port = smtp,465,submission logpath = %(postfix_log)s +backend = %(postfix_backend)s [postfix-rbl] port = smtp,465,submission -logpath = %(syslog_mail)s +logpath = %(postfix_mail)s +backend = %(postfix_backend)s maxretry = 1 @@ -510,12 +523,14 @@ maxretry = 1 port = submission,465,smtp logpath = %(syslog_mail)s +backend = %(syslog_backend)s [sendmail-reject] port = smtp,465,submission logpath = %(syslog_mail)s +backend = %(syslog_backend)s [qmail-rbl] @@ -531,12 +546,14 @@ logpath = /service/qmail/log/main/current port = pop3,pop3s,imap,imaps,submission,465,sieve logpath = %(dovecot_log)s +backend = %(dovecot_backend)s [sieve] port = smtp,465,submission logpath = %(dovecot_log)s +backend = %(dovecot_backend)s [solid-pop3d] @@ -572,6 +589,7 @@ logpath = /opt/kerio/mailserver/store/logs/security.log port = smtp,465,submission,imap3,imaps,pop3,pop3s logpath = %(syslog_mail)s +backend = %(syslog_backend)s [postfix-sasl] @@ -581,12 +599,14 @@ port = smtp,465,submission,imap3,imaps,pop3,pop3s # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. logpath = %(postfix_log)s +backend = %(postfix_backend)s [perdition] port = imap3,imaps,pop3,pop3s logpath = %(syslog_mail)s +backend = %(syslog_backend)s [squirrelmail] @@ -599,12 +619,14 @@ logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log port = imap3,imaps logpath = %(syslog_mail)s +backend = %(syslog_backend)s [uwimap-auth] port = imap3,imaps logpath = %(syslog_mail)s +backend = %(syslog_backend)s # @@ -686,6 +708,7 @@ maxretry = 10 port = 3306 logpath = %(mysql_log)s +backend = %(mysql_backend)s maxretry = 5 @@ -712,12 +735,14 @@ maxretry = 5 # pam-generic filter can be customized to monitor specific subset of 'tty's banaction = iptables-allports logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s [xinetd-fail] banaction = iptables-multiport-log logpath = %(syslog_daemon)s +backend = %(syslog_backend)s maxretry = 2 @@ -748,6 +773,7 @@ action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp enabled = false logpath = %(syslog_daemon)s ; nrpe.cfg may define a different log_facility +backend = %(syslog_backend)s maxretry = 1 diff --git a/config/paths-common.conf b/config/paths-common.conf index bf3cfb6a..7a88e5ce 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -8,8 +8,10 @@ after = paths-overrides.local [DEFAULT] sshd_log = %(syslog_authpriv)s +sshd_backend = auto dropbear_log = %(syslog_authpriv)s +dropbear_backend = auto # There is no sensible generic defaults for syslog log targets, thus # leaving them empty here so that no errors while parsing/interpolating configs @@ -18,6 +20,8 @@ syslog_ftp = syslog_local0 = syslog_mail_warn = syslog_user = +# Set the default syslog backend target to auto +syslog_backend = auto # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log @@ -38,14 +42,17 @@ suhosin_log = %(syslog_user)s %(lighttpd_error_log)s # defaults to ftp or local2 if ftp doesn't exist proftpd_log = %(syslog_ftp)s +proftpd_backend = auto # http://svnweb.freebsd.org/ports/head/ftp/proftpd/files/patch-src_proftpd.8.in?view=markup # defaults to ftp but can be overwritten. pureftpd_log = %(syslog_ftp)s +pureftpd_backend = auto # ftp, daemon and then local7 are tried at configure time however it is overwriteable at configure time # wuftpd_log = %(syslog_ftp)s +wuftpd_backend = auto # syslog_enable defaults to no. so it defaults to vsftpd_log_file setting of /var/log/vsftpd.log # No distro seems to set it to syslog by default @@ -54,13 +61,16 @@ vsftpd_log = /var/log/vsftpd.log # Technically syslog_facility in main.cf can overwrite but no-one sane does this. postfix_log = %(syslog_mail_warn)s +postfix_backend = auto dovecot_log = %(syslog_mail_warn)s +dovecot_backend = auto # Seems to be set at compile time only to LOG_LOCAL0 (src/const.h) at Notice level solidpop3d_log = %(syslog_local0)s mysql_log = %(syslog_daemon)s +mysql_backend = auto roundcube_errors_log = /var/log/roundcube/errors diff --git a/config/paths-fedora.conf b/config/paths-fedora.conf index c5601d3c..b3c978ca 100644 --- a/config/paths-fedora.conf +++ b/config/paths-fedora.conf @@ -37,3 +37,15 @@ exim_main_log = /var/log/exim/main.log mysql_log = /var/lib/mysql/mysqld.log roundcube_errors_log = /var/log/roundcubemail/errors + +# These services will log to the journal via syslog, so use the journal by +# default. +syslog_backend = systemd +sshd_backend = systemd +dropbear_backend = systemd +proftpd_backend = systemd +pureftpd_backend = systemd +wuftpd_backend = systemd +postfix_backend = systemd +dovecot_backend = systemd +mysql_backend = systemd From ced7be94b2f60a035355b774e28eb10560e8ee4d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:43:10 -0600 Subject: [PATCH 080/143] Fix postfix_log typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 36fe6bcf..c288a3fd 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -514,7 +514,7 @@ backend = %(postfix_backend)s [postfix-rbl] port = smtp,465,submission -logpath = %(postfix_mail)s +logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 From 81a26266a9adc438b4a7347a5bd92c7fb29b35f8 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:46:43 -0600 Subject: [PATCH 081/143] Add changlog entry for postfix-rbl logpath change --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 08708a08..e011be9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) + * Use postfix_log logpath for postfix-rbl jail - New Features: From 3a9cf2b3dab47c42b5f4e50b54b31d03b4e93520 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:50:03 -0600 Subject: [PATCH 082/143] Add and use default_backend to set individual backend defaults to auto --- config/paths-common.conf | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/config/paths-common.conf b/config/paths-common.conf index 7a88e5ce..eba6ae4e 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -7,11 +7,13 @@ after = paths-overrides.local [DEFAULT] +default_backend = auto + sshd_log = %(syslog_authpriv)s -sshd_backend = auto +sshd_backend = %(default_backend)s dropbear_log = %(syslog_authpriv)s -dropbear_backend = auto +dropbear_backend = %(default_backend)s # There is no sensible generic defaults for syslog log targets, thus # leaving them empty here so that no errors while parsing/interpolating configs @@ -20,8 +22,8 @@ syslog_ftp = syslog_local0 = syslog_mail_warn = syslog_user = -# Set the default syslog backend target to auto -syslog_backend = auto +# Set the default syslog backend target to default_backend +syslog_backend = %(default_backend)s # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log @@ -42,17 +44,17 @@ suhosin_log = %(syslog_user)s %(lighttpd_error_log)s # defaults to ftp or local2 if ftp doesn't exist proftpd_log = %(syslog_ftp)s -proftpd_backend = auto +proftpd_backend = %(default_backend)s # http://svnweb.freebsd.org/ports/head/ftp/proftpd/files/patch-src_proftpd.8.in?view=markup # defaults to ftp but can be overwritten. pureftpd_log = %(syslog_ftp)s -pureftpd_backend = auto +pureftpd_backend = %(default_backend)s # ftp, daemon and then local7 are tried at configure time however it is overwriteable at configure time # wuftpd_log = %(syslog_ftp)s -wuftpd_backend = auto +wuftpd_backend = %(default_backend)s # syslog_enable defaults to no. so it defaults to vsftpd_log_file setting of /var/log/vsftpd.log # No distro seems to set it to syslog by default @@ -61,16 +63,16 @@ vsftpd_log = /var/log/vsftpd.log # Technically syslog_facility in main.cf can overwrite but no-one sane does this. postfix_log = %(syslog_mail_warn)s -postfix_backend = auto +postfix_backend = %(default_backend)s dovecot_log = %(syslog_mail_warn)s -dovecot_backend = auto +dovecot_backend = %(default_backend)s # Seems to be set at compile time only to LOG_LOCAL0 (src/const.h) at Notice level solidpop3d_log = %(syslog_local0)s mysql_log = %(syslog_daemon)s -mysql_backend = auto +mysql_backend = %(default_backend)s roundcube_errors_log = /var/log/roundcube/errors From 74fcb219ab6d17f365befeb10be99e7a9eb05e16 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Mon, 19 Oct 2015 09:13:05 +0200 Subject: [PATCH 083/143] Enhanced Google domain detection in apache-fakegooglebot Previously, an attacker could fake a domain like crawl-1-1-1-1.googlebot.com.fake.net and get resolved. This change avoids to resolve fake Google domains. --- config/filter.d/ignorecommands/apache-fakegooglebot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 3028d86a..9e0f0d83 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,7 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - if not host or not re.match('crawl-.*\.googlebot\.com', host): + if not host or not re.match('crawl-.*\.googlebot\.com$', host): sys.exit(1) host_ips = DNSUtils.dnsToIp(host) sys.exit(0 if ip in host_ips else 1) From 2c576c64f8192cad469bd839321c03bcc1f8113a Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Tue, 20 Oct 2015 10:37:07 +0200 Subject: [PATCH 084/143] Change domain filter regex Change domain filter regex since there are other Google crawlers. See "Google crawlers" --- ChangeLog | 3 +++ config/filter.d/ignorecommands/apache-fakegooglebot | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2f5a158b..9449f2cd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) + * Fix jail.conf.5 man's section (gh-1226) - New Features: @@ -34,6 +35,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks M. Maraun) * Added check against atacker's Googlebot PTR fake records (Thanks Pablo Rodriguez Fernandez) + * Enhance filter against atacker's Googlebot PTR fake records + (gh-1226) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 9e0f0d83..19fb5107 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,7 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - if not host or not re.match('crawl-.*\.googlebot\.com$', host): + if not host or not re.match('.*\.google(bot)?\.com$', host): sys.exit(1) host_ips = DNSUtils.dnsToIp(host) sys.exit(0 if ip in host_ips else 1) From 2861a957a9ba691a44005673e90883fd1def1890 Mon Sep 17 00:00:00 2001 From: 1technophile Date: Sun, 25 Oct 2015 20:36:40 +0100 Subject: [PATCH 085/143] filter for openhab domotic software authentication failure with the rest api and web interface + test cases; closes gh-1223 --- config/filter.d/openhab.conf | 16 ++++++++++++++++ config/jail.conf | 6 ++++++ fail2ban/tests/files/logs/openhab | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 config/filter.d/openhab.conf create mode 100644 fail2ban/tests/files/logs/openhab diff --git a/config/filter.d/openhab.conf b/config/filter.d/openhab.conf new file mode 100644 index 00000000..83857c7a --- /dev/null +++ b/config/filter.d/openhab.conf @@ -0,0 +1,16 @@ +# Openhab brute force auth filter: /etc/fail2ban/filter.d/openhab.conf: +# +# Block IPs trying to auth openhab by web or rest api +# +# Matches e.g. +# 12.34.33.22 - - [26/sept./2015:18:04:43 +0200] "GET /openhab.app HTTP/1.1" 401 1382 +# 175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /rest/bindings HTTP/1.1" 401 1384 + +[Definition] +failregex = ^\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$ + +[Init] +datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z + + + diff --git a/config/jail.conf b/config/jail.conf index 7500f4ff..fd7f376e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -306,6 +306,12 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 +[openhab-auth] + +filter = openhab +action = iptables-allports[name=NoAuthFailures] +logpath = /opt/openhab/logs/request.log + [nginx-http-auth] port = http,https diff --git a/fail2ban/tests/files/logs/openhab b/fail2ban/tests/files/logs/openhab new file mode 100644 index 00000000..983989a9 --- /dev/null +++ b/fail2ban/tests/files/logs/openhab @@ -0,0 +1,11 @@ +# should match +# failJSON: { "time": "2015-09-02T00:11:31", "match": true , "host": "175.18.15.10" } +175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /openhab.app HTTP/1.1" 401 1382 +# failJSON: { "time": "2015-09-02T00:11:31", "match": true , "host": "175.18.15.10" } +175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /rest/bindings HTTP/1.1" 401 1384 + +# Should not match +# failJSON: { "match": false } +175.18.15.11 - - [17/oct./2015:00:35:12 +0200] "GET /openhab.app?sitemap=default&poll=true&__async=true&__source=waHome HTTP/1.1" 200 92 +# failJSON: { "match": false } +175.18.15.11 - - [16/oct./2015:20:29:38 +0200] "GET /rest/sitemaps/default/maison HTTP/1.1" 200 2837 From eb87638eadbfe215f875675b0f2383222830f424 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 26 Oct 2015 15:52:10 +0100 Subject: [PATCH 086/143] ChangeLog entry for OpenHAB home automation filter (gh-1223) --- ChangeLog | 2 ++ THANKS | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9449f2cd..391eabc6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) + * Added filter for openhab domotic software authentication failure with the + rest api and web interface (gh-1223) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/THANKS b/THANKS index 45674847..47156d82 100644 --- a/THANKS +++ b/THANKS @@ -40,6 +40,7 @@ Eric Gerbier Enrico Labedzki Eugene Hopkinson (SlowRiot) ftoppi +Florian Robert (1technophile) François Boulogne Frantisek Sumsal Frédéric From 6884593ab825068cdecc55658a9213513ca6ed16 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:15:20 +0100 Subject: [PATCH 087/143] New filter `nginx-limit-req` ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) --- ChangeLog | 7 ++-- config/filter.d/nginx-limit-req.conf | 39 +++++++++++++++++++++++ config/jail.conf | 8 +++++ fail2ban/tests/files/logs/nginx-limit-req | 6 ++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 config/filter.d/nginx-limit-req.conf create mode 100644 fail2ban/tests/files/logs/nginx-limit-req diff --git a/ChangeLog b/ChangeLog index 391eabc6..cd1ce7bd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,11 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) - New Features: + * New filters: + - openhab - domotic software authentication failure with the + rest api and web interface (gh-1223) + - nginx-limit-req - ban hosts, that were failed through nginx by limit + request processing rate (ngx_http_limit_req_module) - Enhancements: * Do not rotate empty log files @@ -37,8 +42,6 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) - * Added filter for openhab domotic software authentication failure with the - rest api and web interface (gh-1223) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf new file mode 100644 index 00000000..eb804798 --- /dev/null +++ b/config/filter.d/nginx-limit-req.conf @@ -0,0 +1,39 @@ +# Fail2ban filter configuration for nginx :: limit_req +# used to ban hosts, that were failed through nginx by limit request processing rate +# +# Author: Serg G. Brester (sebres) +# +# To use 'nginx-limit-req' filter you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# +# Example: +# +# http { +# ... +# limit_req_zone $binary_remote_addr zone=lr_zone:10m rate=1r/s; +# ... +# # http, server, or location: +# location ... { +# limit_req zone=lr_zone burst=1 nodelay; +# ... +# } +# ... +# } +# ... +# + +[Definition] + +# Specify following expression to define exact zones, if you want to ban IPs limited +# from specified zones only. +# Example: +# +# ngx_limit_req_zones = lr_zone|lr_zone2 +# +ngx_limit_req_zones = [^"]+ + +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +ignoreregex = + diff --git a/config/jail.conf b/config/jail.conf index fd7f376e..fc175550 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -317,6 +317,14 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# or for example see in 'config/filter.d/nginx-limit-req.conf' +[nginx-limit-req] +port = http,https +logpath = %(nginx_error_log)s + [nginx-botsearch] port = http,https diff --git a/fail2ban/tests/files/logs/nginx-limit-req b/fail2ban/tests/files/logs/nginx-limit-req new file mode 100644 index 00000000..68f1b239 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-limit-req @@ -0,0 +1,6 @@ + +# failJSON: { "time": "2015-10-29T20:01:02", "match": true , "host": "1.2.3.4" } +2015/10/29 20:01:02 [error] 256554#0: *99927 limiting requests, excess: 1.852 by zone "one", client: 1.2.3.4, server: example.com, request: "POST /index.htm HTTP/1.0", host: "exmaple.com" + +# failJSON: { "time": "2015-10-29T19:24:05", "match": true , "host": "192.0.2.0" } +2015/10/29 19:24:05 [error] 12684#12684: *22174 limiting requests, excess: 1.495 by zone "one", client: 192.0.2.0, server: example.com, request: "GET /index.php HTTP/1.1", host: "example.com", referrer: "https://example.com" From 53b39162a1b2d026bf0e9eb14f3ffa61f5cc1f8a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:55:23 +0100 Subject: [PATCH 088/143] Shortly, much faster and stable version of regexp (possible because expression is start-anchored and does not contains closely to catch-all sub expressions) --- config/filter.d/nginx-limit-req.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index eb804798..589d3d78 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -33,7 +33,13 @@ # ngx_limit_req_zones = [^"]+ -failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ +# Use following full expression if you should range limit request to specified +# servers, requests, referrers etc. only : +# +# failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +# Shortly, much faster and stable version of regexp: +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: ignoreregex = From f359ed8c367a97060afcc54164263bfad9ed92d9 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 15:36:18 +0100 Subject: [PATCH 089/143] Fixed directly defined banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (+ man entries for both variables added); closes gh-1216 --- ChangeLog | 2 ++ config/jail.conf | 7 ++++--- man/jail.conf.5 | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..beaba762 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) * Fix jail.conf.5 man's section (gh-1226) + * Fixed default banaction for allports jails like pam-generic, recidive, etc + with new default variable `banaction_allports` (gh-1216) - New Features: * New filters: diff --git a/config/jail.conf b/config/jail.conf index fc175550..69fef818 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -154,6 +154,7 @@ port = 0:65535 # action_* variables. Can be overridden globally or per # section within jail.local file banaction = iptables-multiport +banaction_allports = iptables-allports # The simplest action to take: ban only action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] @@ -713,7 +714,7 @@ maxretry = 5 [recidive] logpath = /var/log/fail2ban.log -banaction = iptables-allports +banaction = %(banaction_allports)s bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 @@ -724,7 +725,7 @@ maxretry = 5 [pam-generic] # pam-generic filter can be customized to monitor specific subset of 'tty's -banaction = iptables-allports +banaction = %(banaction_allports)s logpath = %(syslog_authpriv)s @@ -770,7 +771,7 @@ maxretry = 1 enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current maxretry = 6 -banaction = iptables-allports +banaction = %(banaction_allports)s [directadmin] enabled = false diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 957a04b4..5dc64a4b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -146,6 +146,12 @@ Ensure syslog or the program that generates the log file isn't configured to com .B logencoding encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP +.B banaction +default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +.TP +.B banaction_allports +default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +.TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS From e825e977ccb897ddeadd573854e2666ee8963038 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 17:51:30 +0100 Subject: [PATCH 090/143] Nginx log paths extended (prefixed with "*" wildcard) closes gh-1237 --- ChangeLog | 1 + config/paths-common.conf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..66805c36 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) + * Nginx log paths extended (prefixed with "*" wildcard) (gh-1237) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/paths-common.conf b/config/paths-common.conf index bf3cfb6a..1aac027b 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -24,9 +24,9 @@ auditd_log = /var/log/audit/audit.log exim_main_log = /var/log/exim/mainlog -nginx_error_log = /var/log/nginx/error.log +nginx_error_log = /var/log/nginx/*error.log -nginx_access_log = /var/log/nginx/access.log +nginx_access_log = /var/log/nginx/*access.log lighttpd_error_log = /var/log/lighttpd/error.log From fcf03790f4377f4083ca114fffc8aea5e67685ee Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 1 Nov 2015 16:55:45 +0100 Subject: [PATCH 091/143] fixed misleading documentation of `banaction` --- man/jail.conf.5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5dc64a4b..51ea7097 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -147,10 +147,10 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports -default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: From b40c6cbd9a0c1b51e0cc0f0c4629b84d60c2c129 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 1 Nov 2015 11:28:58 -0500 Subject: [PATCH 092/143] ENH: .mailmap file to bring some names together for git shortlog -sn --- .mailmap | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..9b7cce3a --- /dev/null +++ b/.mailmap @@ -0,0 +1,5 @@ +Lee Clemens +Serg G. Brester +Serg G. Brester +Serg G. Brester +Viktor Szépe From 94cffece12dc2ff0efd6a8b6761b079eb9491c68 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 2 Nov 2015 21:19:15 +0100 Subject: [PATCH 093/143] New interpolation feature for definition config readers - ``, as extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters; --- ChangeLog | 7 ++++ fail2ban/client/configreader.py | 4 +- fail2ban/tests/clientreadertestcase.py | 57 ++++++++++++++++++-------- man/jail.conf.5 | 30 ++++++++++++-- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..151b5a9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,13 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released with new default variable `banaction_allports` (gh-1216) - New Features: + * New interpolation feature for definition config readers - `` + (means last known init definition of filters or actions with name `parameter`). + This interpolation makes possible to extend a parameters of stock filter or + action directly in jail inside jail.local file, without creating a separately + filter.d/*.local file. + As extension to interpolation `%(known/parameter)s`, that does not works for + filter and action init parameters * New filters: - openhab - domotic software authentication failure with the rest api and web interface (gh-1223) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d5675cc8..c6dd1b60 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -285,8 +285,10 @@ class DefinitionInitConfigReader(ConfigReader): if self.has_section("Init"): for opt in self.options("Init"): + v = self.get("Init", opt) + self._initOpts['known/'+opt] = v if not opt in self._initOpts: - self._initOpts[opt] = self.get("Init", opt) + self._initOpts[opt] = v def convert(self): raise NotImplementedError diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 94fe1828..d19090be 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -165,11 +165,11 @@ class JailReaderTest(LogCaptureTestCase): self.__share_cfg = {} def testIncorrectJail(self): - jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg) + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg) self.assertRaises(ValueError, jail.read) def testJailActionEmpty(self): - jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -177,7 +177,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): - jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -200,7 +200,7 @@ class JailReaderTest(LogCaptureTestCase): if STOCK: def testStockSSHJail(self): - jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -274,6 +274,10 @@ class JailReaderTest(LogCaptureTestCase): class FilterReaderTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(FilterReaderTest, self).__init__(*args, **kwargs) + self.__share_cfg = {} + def testConvert(self): output = [['set', 'testcase01', 'addfailregex', "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" @@ -311,9 +315,8 @@ class FilterReaderTest(unittest.TestCase): # is unreliable self.assertEqual(sorted(filterReader.convert()), sorted(output)) - filterReader = FilterReader( - "testcase01", "testcase01", {'maxlines': "5"}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) @@ -322,8 +325,8 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -331,16 +334,34 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertEqual(sorted(c), sorted(output)) + + def testFilterReaderSubstitionKnown(self): + output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=']] + filterName, filterOpt = JailReader.extractOptions( + 'substition[honeypot=",", sweet="test,,test2"]') + filterReader = FilterReader('substition', "jailname", filterOpt, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() self.assertEqual(sorted(c), sorted(output)) def testFilterReaderSubstitionFail(self): - filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}) - filterReader.setBaseDir(TEST_FILES_DIR) + # directly subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + self.assertRaises(ValueError, FilterReader.convert, filterReader) + # cross subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) @@ -508,12 +529,13 @@ class JailsReaderTest(LogCaptureTestCase): if jail == 'INCLUDES': continue filterName = jails.get(jail, 'filter') + filterName, filterOpt = JailReader.extractOptions(filterName) allFilters.add(filterName) self.assertTrue(len(filterName)) # moreover we must have a file for it # and it must be readable as a Filter - filterReader = FilterReader(filterName, jail, {}) - filterReader.setBaseDir(CONFIG_DIR) + filterReader = FilterReader(filterName, jail, filterOpt, + share_config=self.__share_cfg, basedir=CONFIG_DIR) self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName) # opens fine filterReader.getOptions({}) # reads fine @@ -551,7 +573,10 @@ class JailsReaderTest(LogCaptureTestCase): filters = set(os.path.splitext(os.path.split(a)[1])[0] for a in glob.glob(os.path.join('config', 'filter.d', '*.conf')) if not a.endswith('common.conf')) - filters_jail = set(jail.options['filter'] for jail in jails.jails) + # get filters of all jails (filter names without options inside filter[...]) + filters_jail = set( + JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails + ) self.maxDiff = None self.assertTrue(filters.issubset(filters_jail), "More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail)) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 51ea7097..7a31e8b1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -89,13 +89,33 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. .RS +.nf baduseragents = IE|wget +failregex = %(known/failregex)s + useragent=%(baduseragents)s +.fi .RE + +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. + .RS -failregex = useragent=%(baduseragents)s +# filter.d/test.conf: +.nf +[Init] +test.method = GET +baduseragents = IE|wget +[Definition] +failregex = ^%(__prefix_line)\\s+""\\s+test\\s+regexp\\s+-\\s+useragent=(?:) + +# jail.local: +[test] +# use filter "test", overwrite method to "POST" and extend known bad agents with "badagent": +filter = test[test.method=POST, baduseragents="badagent|"] +.fi .RE Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. @@ -253,7 +273,7 @@ The maximum period of time in seconds that a command can executed, before being Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed. @@ -312,7 +332,7 @@ is the regex to identify log entries that should be ignored by Fail2Ban, even if .PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. @@ -327,6 +347,8 @@ Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO860 \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP +Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. + Filters can also have a section called [INCLUDES]. This is used to read other configuration files. .TP From ba76f4ca2fa058ca187a2990d3ff0b66b6211940 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 2 Nov 2015 15:21:14 -0700 Subject: [PATCH 094/143] Fix typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 69fef818..3bb17e0a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -80,7 +80,7 @@ maxretry = 5 # auto: will try to use the following backends, in order: # pyinotify, gamin, polling. # -# Note: if systemd backend is choses as the default but you enable a jail +# Note: if systemd backend is chosen as the default but you enable a jail # for which logs are present only in its own log files, specify some other # backend for that jail (e.g. polling) and provide empty value for # journalmatch. See https://github.com/fail2ban/fail2ban/issues/959#issuecomment-74901200 From a42aa726ab22d07980ba811bf09151c1e49ce2a4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 20:13:03 +0100 Subject: [PATCH 095/143] fixed fail2ban-regex reads invalid character (in sense of given encoding); continuing to process line ignoring invalid characters (still has no test cases). filter test cases added for same issue inside fail2ban-server / fail2ban-testcases; closes gh-1248 --- bin/fail2ban-regex | 10 +++++-- fail2ban/tests/filtertestcase.py | 49 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index b7b0579f..fdbfcb7a 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -84,8 +84,14 @@ def file_lines_gen(hdlr): try: line = line.decode(fail2banRegex.encoding, 'strict') except UnicodeDecodeError: - if sys.version_info >= (3,): # Python 3 must be decoded - line = line.decode(fail2banRegex.encoding, 'ignore') + logSys.warning( + "Error decoding line from '%s' with '%s'." + " Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r" % + ('', fail2banRegex.encoding, line)) + # decode with replacing error chars: + line = line.decode(fail2banRegex.encoding, 'replace') yield line def journal_lines_gen(myjournal): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index acf15528..b15494f5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -90,7 +90,11 @@ def _assert_equal_entries(utest, found, output, count=None): found_time, output_time = \ MyTime.localtime(found[2]),\ MyTime.localtime(output[2]) - utest.assertEqual(found_time, output_time) + try: + utest.assertEqual(found_time, output_time) + except AssertionError as e: + # assert more structured: + utest.assertEqual((float(found[2]), found_time), (float(output[2]), output_time)) if len(output) > 3 and count is None: # match matches # do not check if custom count (e.g. going through them twice) if os.linesep != '\n' or sys.platform.startswith('cygwin'): @@ -216,6 +220,14 @@ class BasicFilter(unittest.TestCase): ("^%Y-%m-%d-%H%M%S.%f %z", "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset")) + def testAssertWrongTime(self): + self.assertRaises(AssertionError, + lambda: _assert_equal_entries(self, + ('1.1.1.1', 1, 1421262060.0), + ('1.1.1.1', 1, 1421262059.0), + 1) + ) + class IgnoreIP(LogCaptureTestCase): @@ -900,6 +912,41 @@ class GetFailures(unittest.TestCase): except FailManagerEmpty: pass + def testGetFailuresWrongChar(self): + # write wrong utf-8 char: + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + fout = fopen(fname, 'wb') + try: + # write: + for l in ( + b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char + b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars + b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars + ): + fout.write(l) + fout.close() + # + output = ('192.0.2.0', 3, 1421262060.0) + failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" + + # encoding - auto + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + # test direct set of encoding: + for enc in ('utf-8', 'ascii'): + self.tearDown();self.setUp(); + self.filter.setLogEncoding('utf-8'); + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + finally: + _killfile(fout, fname) + def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ('93.184.216.34', 2, 1124013539.0, From 46b116e86abb37428220972dfb064d0a8d101a67 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 21:52:06 +0100 Subject: [PATCH 096/143] filter test cases improved + log captured inside such tests + python 3.x compatibility; changelog entry; --- ChangeLog | 2 ++ fail2ban/tests/filtertestcase.py | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..862f46f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) * Fixed default banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (gh-1216) + * Fixed `fail2ban-regex` stops working on invalid (wrong encoded) character + for python version < 3.x (gh-1248) - New Features: * New filters: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b15494f5..3674a574 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -822,7 +822,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover return MonitorJournalFailures -class GetFailures(unittest.TestCase): +class GetFailures(LogCaptureTestCase): FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") @@ -837,6 +837,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) @@ -848,6 +849,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" tearDownMyTime() + LogCaptureTestCase.tearDown(self) def testTail(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) @@ -929,20 +931,20 @@ class GetFailures(unittest.TestCase): output = ('192.0.2.0', 3, 1421262060.0) failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" - # encoding - auto - self.filter.addLogPath(fname) - self.filter.addFailRegex(failregex) - self.filter.getFailures(fname) - _assert_correct_last_attempt(self, self.filter, output) - - # test direct set of encoding: - for enc in ('utf-8', 'ascii'): - self.tearDown();self.setUp(); - self.filter.setLogEncoding('utf-8'); + # test encoding auto or direct set of encoding: + for enc in (None, 'utf-8', 'ascii'): + if enc is not None: + self.tearDown();self.setUp(); + self.filter.setLogEncoding(enc); + self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); finally: _killfile(fout, fname) From 0877d662287bb74cb403c0159841dbbafda28c08 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:46:19 +0100 Subject: [PATCH 097/143] fail2ban-regex moved to the client + test cases for initial coverage added --- .../client/fail2banregex.py | 203 +++++++++--------- fail2ban/server/filter.py | 18 +- fail2ban/tests/fail2banregextestcase.py | 155 +++++++++++++ fail2ban/tests/files/testcase-wrong-char.log | 4 + fail2ban/tests/utils.py | 4 + 5 files changed, 275 insertions(+), 109 deletions(-) rename bin/fail2ban-regex => fail2ban/client/fail2banregex.py (81%) create mode 100644 fail2ban/tests/fail2banregextestcase.py create mode 100644 fail2ban/tests/files/testcase-wrong-char.log diff --git a/bin/fail2ban-regex b/fail2ban/client/fail2banregex.py similarity index 81% rename from bin/fail2ban-regex rename to fail2ban/client/fail2banregex.py index fdbfcb7a..12dde7f1 100755 --- a/bin/fail2ban-regex +++ b/fail2ban/client/fail2banregex.py @@ -44,16 +44,16 @@ from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderErro try: from systemd import journal - from fail2ban.server.filtersystemd import FilterSystemd + from ..server.filtersystemd import FilterSystemd except ImportError: journal = None -from fail2ban.version import version -from fail2ban.client.filterreader import FilterReader -from fail2ban.server.filter import Filter -from fail2ban.server.failregex import RegexException +from ..version import version +from .filterreader import FilterReader +from ..server.filter import Filter, FileContainer +from ..server.failregex import RegexException -from fail2ban.helpers import FormatterWithTraceBack, getLogger +from ..helpers import FormatterWithTraceBack, getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") @@ -63,6 +63,9 @@ def debuggexURL(sample, regex): 'flavor': 'python' }) return 'http://www.debuggex.com/?' + q +def output(args): + print(args) + def shortstr(s, l=53): """Return shortened string """ @@ -77,22 +80,7 @@ def pprint_list(l, header=None): s = "|- %s\n" % header else: s = '' - print s + "| " + "\n| ".join(l) + '\n`-' - -def file_lines_gen(hdlr): - for line in hdlr: - try: - line = line.decode(fail2banRegex.encoding, 'strict') - except UnicodeDecodeError: - logSys.warning( - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r" % - ('', fail2banRegex.encoding, line)) - # decode with replacing error chars: - line = line.decode(fail2banRegex.encoding, 'replace') - yield line + output( s + "| " + "\n| ".join(l) + '\n`-' ) def journal_lines_gen(myjournal): while True: @@ -259,14 +247,14 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - print "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) + output( "Use datepattern : %s" % ( + self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: self._filter.setMaxLines(int(v)) self._maxlines_set = True - print "Use maxlines : %d" % self._filter.getMaxLines() + output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): if self._journalmatch is None: @@ -280,18 +268,18 @@ class Fail2banRegex(object): ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): basedir = os.path.dirname(os.path.dirname(value)) value = os.path.splitext(os.path.basename(value))[0] - print "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) + output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir) if not reader.read(): - print "ERROR: failed to load filter %s" % value + output( "ERROR: failed to load filter %s" % value ) return False else: ## foreign file - readexplicit this file and includes if possible: - print "Use %11s file : %s" % (regex, value) + output( "Use %11s file : %s" % (regex, value) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config) reader.setBaseDir(None) if not reader.readexplicit(): - print "ERROR: failed to read %s" % value + output( "ERROR: failed to read %s" % value ) return False reader.getOptions(None) readercommands = reader.convert() @@ -307,8 +295,8 @@ class Fail2banRegex(object): try: self.setMaxLines(maxlines) except ValueError: - print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ - "read from %(value)s" % locals() + output( "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() ) return False elif command[2] == 'addjournalmatch': journalmatch = command[3:] @@ -317,7 +305,7 @@ class Fail2banRegex(object): datepattern = command[3] self.setDatePattern(datepattern) else: - print "Use %11s line : %s" % (regex, shortstr(value)) + output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = [RegexStat(value)] setattr(self, "_" + regex, regex_values) @@ -335,7 +323,7 @@ class Fail2banRegex(object): found = True regex = self._ignoreregex[ret].inc() except RegexException, e: - print e + output( e ) return False return found @@ -352,10 +340,10 @@ class Fail2banRegex(object): regex.inc() regex.appendIP(match) except RegexException, e: - print e + output( e ) return False except IndexError: - print "Sorry, but no found in regex" + output( "Sorry, but no found in regex" ) return False for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -376,7 +364,7 @@ class Fail2banRegex(object): t0 = time.time() for line_no, line in enumerate(test_lines): if isinstance(line, tuple): - line_datetimestripped, ret = fail2banRegex.testRegex( + line_datetimestripped, ret = self.testRegex( line[0], line[1]) line = "".join(line[0]) else: @@ -384,8 +372,8 @@ class Fail2banRegex(object): if line.startswith('#') or not line: # skip comment and empty lines continue - line_datetimestripped, ret = fail2banRegex.testRegex(line) - is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped) + line_datetimestripped, ret = self.testRegex(line) + is_ignored = self.testIgnoreRegex(line_datetimestripped) if is_ignored: self._line_stats.ignored += 1 @@ -432,18 +420,18 @@ class Fail2banRegex(object): b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) elif lines < self._maxlines or getattr(self, '_print_all_' + ltype): pprint_list([x.rstrip() for x in l], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) def printStats(self): - print - print "Results" - print "=======" + output( "" ) + output( "Results" ) + output( "=======" ) def print_failregexes(title, failregexes): # Print title @@ -464,7 +452,7 @@ class Fail2banRegex(object): timeString, ip[-1] and " (multiple regex matched)" or "")) - print "\n%s: %d total" % (title, total) + output( "\n%s: %d total" % (title, total) ) pprint_list(out, " #) [# of hits] regular expression") return total @@ -474,7 +462,7 @@ class Fail2banRegex(object): if self._filter.dateDetector is not None: - print "\nDate template hits:" + output( "\nDate template hits:" ) out = [] for template in self._filter.dateDetector.templates: if self._verbose or template.hits: @@ -482,10 +470,10 @@ class Fail2banRegex(object): template.hits, template.name)) pprint_list(out, "[# of hits] date format") - print "\nLines: %s" % self._line_stats, + output( "\nLines: %s" % self._line_stats, ) if self._time_elapsed is not None: - print "[processed in %.2f sec]" % self._time_elapsed, - print + output( "[processed in %.2f sec]" % self._time_elapsed, ) + output( "" ) if self._print_all_matched: self.printLines('matched') @@ -496,9 +484,62 @@ class Fail2banRegex(object): return True + def file_lines_gen(self, hdlr): + for line in hdlr: + yield FileContainer.decode_line('', self.encoding, line) -if __name__ == "__main__": + def start(self, opts, args): + cmd_log, cmd_regex = args[:2] + + if not self.readRegex(cmd_regex, 'fail'): + return False + + if len(args) == 3 and not self.readRegex(args[2], 'ignore'): + return False + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log, 'rb') + output( "Use log file : %s" % cmd_log ) + output( "Use encoding : %s" % self.encoding ) + test_lines = self.file_lines_gen(hdlr) + except IOError, e: + output( e ) + return False + elif cmd_log == "systemd-journal": + if not journal: + output( "Error: systemd library not found. Exiting..." ) + return False + myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + journalmatch = self._journalmatch + self.setDatePattern(None) + if journalmatch: + try: + for element in journalmatch: + if element == "+": + myjournal.add_disjunction() + else: + myjournal.add_match(element) + except ValueError: + output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) + return False + output( "Use journal match : %s" % " ".join(journalmatch) ) + test_lines = journal_lines_gen(myjournal) + else: + output( "Use single line : %s" % shortstr(cmd_log) ) + test_lines = [ cmd_log ] + output( "" ) + + self.process(test_lines) + + if not self.printStats(): + return False + + return True + + +def exec_command_line(): # pragma: no cover parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -510,18 +551,16 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) - print - print "Running tests" - print "=============" - print - - fail2banRegex = Fail2banRegex(opts) - # We need 2 or 3 parameters if not len(args) in (2, 3): sys.stderr.write("ERROR: provide both and .\n\n") parser.print_help() - sys.exit(-1) + return False + + output( "" ) + output( "Running tests" ) + output( "=============" ) + output( "" ) # TODO: taken from -testcases -- move common functionality somewhere if opts.log_level is not None: # pragma: no cover @@ -552,46 +591,6 @@ if __name__ == "__main__": stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) - cmd_log, cmd_regex = args[:2] - - fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) - - if len(args) == 3: - fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) - - if os.path.isfile(cmd_log): - try: - hdlr = open(cmd_log, 'rb') - print "Use log file : %s" % cmd_log - print "Use encoding : %s" % fail2banRegex.encoding - test_lines = file_lines_gen(hdlr) - except IOError, e: - print e - sys.exit(-1) - elif cmd_log == "systemd-journal": - if not journal: - print "Error: systemd library not found. Exiting..." - sys.exit(-1) - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) - journalmatch = fail2banRegex._journalmatch - fail2banRegex.setDatePattern(None) - if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - print "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) - sys.exit(-1) - print "Use journal match : %s" % " ".join(journalmatch) - test_lines = journal_lines_gen(myjournal) - else: - print "Use single line : %s" % shortstr(cmd_log) - test_lines = [ cmd_log ] - print - - fail2banRegex.process(test_lines) - - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex = Fail2banRegex(opts) + if not fail2banRegex.start(opts, args): + sys.exit(-1) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6fc2cd6c..65be0467 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -792,23 +792,27 @@ class FileContainer: self.__handler.seek(self.__pos) return True - def readline(self): - if self.__handler is None: - return "" - line = self.__handler.readline() + @staticmethod + def decode_line(filename, enc, line): try: - line = line.decode(self.getEncoding(), 'strict') + line = line.decode(enc, 'strict') except UnicodeDecodeError: logSys.warning( "Error decoding line from '%s' with '%s'." " Consider setting logencoding=utf-8 (or another appropriate" " encoding) for this jail. Continuing" " to process line ignoring invalid characters: %r" % - (self.getFileName(), self.getEncoding(), line)) + (filename, enc, line)) # decode with replacing error chars: - line = line.decode(self.getEncoding(), 'replace') + line = line.decode(enc, 'replace') return line + def readline(self): + if self.__handler is None: + return "" + return FileContainer.decode_line( + self.getFileName(), self.getEncoding(), self.__handler.readline()) + def close(self): if not self.__handler is None: # Saves the last position. diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py new file mode 100644 index 00000000..ee10128e --- /dev/null +++ b/fail2ban/tests/fail2banregextestcase.py @@ -0,0 +1,155 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__author__ = "Serg Brester" +__copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors" +__license__ = "GPL" + +from __builtin__ import open as fopen +import unittest +import getpass +import os +import sys +import time +import tempfile +import uuid + +try: + from systemd import journal +except ImportError: + journal = None + +from ..client import fail2banregex +from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output +from .utils import LogCaptureTestCase, logSys + + +fail2banregex.logSys = logSys +def _test_output(*args): + logSys.info(args[0]) + +fail2banregex.output = _test_output + +CONF_FILES_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__),"..", "..", "config")) +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + + +def _Fail2banRegex(*args): + parser = get_opt_parser() + (opts, args) = parser.parse_args(list(args)) + return (opts, args, Fail2banRegex(opts)) + +class Fail2banRegexTest(LogCaptureTestCase): + + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") + FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") + FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") + + FILTER_SSHD = os.path.join(CONF_FILES_DIR, 'filter.d', 'sshd.conf') + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testWrongRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".** from $" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testWrongIngnoreRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".*? from $", r".**" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testDirectFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", "--print-no-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + + def testDirectNotFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"XYZ from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed') + + def testDirectIgnored(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-ignored", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $", + r"kevin from 192.0.2.0$" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed') + + def testDirectRE_1(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_01, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters') + + self.assertLogged('Dez 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128') + self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10') + + def testDirectRE_2(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_02, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + def testWronChar(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); + + self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') + self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') diff --git a/fail2ban/tests/files/testcase-wrong-char.log b/fail2ban/tests/files/testcase-wrong-char.log new file mode 100644 index 00000000..9736020e --- /dev/null +++ b/fail2ban/tests/files/testcase-wrong-char.log @@ -0,0 +1,4 @@ +Nov 8 00:16:12 main sshd[32547]: Invalid user llinco\361ir from 192.0.2.0 +Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco\361ir +Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llincoñir +Nov 8 00:16:14 main sshd[32547]: Failed password for invalid user llinco\361ir from 192.0.2.0 port 57025 ssh2 diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 35fa59fd..8172e7ec 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -85,6 +85,7 @@ def gatherTests(regexps=None, no_network=False): from . import misctestcase from . import databasetestcase from . import samplestestcase + from . import fail2banregextestcase if not regexps: # pragma: no cover tests = unittest.TestSuite() @@ -152,6 +153,9 @@ def gatherTests(regexps=None, no_network=False): # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) + # bin/fail2ban-regex + tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest)) + # # Python action testcases # From 38f09b417ad514254b81edb210322d6658c5a9e9 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:48:37 +0100 Subject: [PATCH 098/143] fail2ban-regex command line (after fail2ban-regex functionality moved to the client) --- bin/fail2ban-regex | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 bin/fail2ban-regex diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex new file mode 100755 index 00000000..584c1ea7 --- /dev/null +++ b/bin/fail2ban-regex @@ -0,0 +1,34 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" +__license__ = "GPL" + +from fail2ban.client.fail2banregex import exec_command_line + +exec_command_line() From 689dfa1e6a27aa49e0c06895d7cb049e6352174c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 12:19:46 +0100 Subject: [PATCH 099/143] debuggexURL fixed for wrong encoded character; test cases extended; --- fail2ban/client/fail2banregex.py | 15 ++++++++----- fail2ban/tests/fail2banregextestcase.py | 30 +++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 12dde7f1..92ad4a91 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -204,7 +204,7 @@ class LineStats(object): # just for convenient str def __getitem__(self, key): - return getattr(self, key) + return getattr(self, key) if hasattr(self, key) else '' class Fail2banRegex(object): @@ -240,7 +240,11 @@ class Fail2banRegex(object): else: self.encoding = locale.getpreferredencoding() + def decode_line(self, line): + return FileContainer.decode_line('', self.encoding, line) + def encode_line(self, line): + return line.encode(self.encoding, 'ignore') def setDatePattern(self, pattern): if not self._datepattern_set: @@ -398,8 +402,6 @@ class Fail2banRegex(object): self._filter.dateDetector.sortTemplate() self._time_elapsed = time.time() - t0 - - def printLines(self, ltype): lstats = self._line_stats assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) @@ -417,7 +419,8 @@ class Fail2banRegex(object): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) + b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ @@ -486,7 +489,7 @@ class Fail2banRegex(object): def file_lines_gen(self, hdlr): for line in hdlr: - yield FileContainer.decode_line('', self.encoding, line) + yield self.decode_line(line) def start(self, opts, args): @@ -507,7 +510,7 @@ class Fail2banRegex(object): except IOError, e: output( e ) return False - elif cmd_log == "systemd-journal": + elif cmd_log == "systemd-journal": # pragma: no cover if not journal: output( "Error: systemd library not found. Exiting..." ) return False diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index ee10128e..2fd362c7 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -60,6 +60,8 @@ def _Fail2banRegex(*args): class Fail2banRegexTest(LogCaptureTestCase): + RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") @@ -120,7 +122,7 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_01, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') @@ -135,11 +137,23 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_02, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + def testVerbose(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--verbose", "--print-no-missed", + Fail2banRegexTest.FILENAME_02, + Fail2banRegexTest.RE_00 + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + self.assertLogged('141.3.81.106 Fri Aug 14 11:53:59 2015') + self.assertLogged('141.3.81.106 Fri Aug 14 11:54:59 2015') + def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD @@ -153,3 +167,15 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') + + def testWronCharDebuggex(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--debuggex", "--print-all-matched", + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('http://') + + From 441dffbe2a539a3910c5ae4fc4e1341d9a103640 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:31:56 -0500 Subject: [PATCH 100/143] ENH: Pruned some "pragma: no cover"s in fail2banregex This code should and can be unit-tested, so no reason to keep it with no cover --- fail2ban/client/fail2banregex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 92ad4a91..d0cdab84 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -542,7 +542,7 @@ class Fail2banRegex(object): return True -def exec_command_line(): # pragma: no cover +def exec_command_line(): parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -566,10 +566,10 @@ def exec_command_line(): # pragma: no cover output( "" ) # TODO: taken from -testcases -- move common functionality somewhere - if opts.log_level is not None: # pragma: no cover + if opts.log_level is not None: # so we had explicit settings logSys.setLevel(getattr(logging, opts.log_level.upper())) - else: # pragma: no cover + else: # suppress the logging but it would leave unittests' progress dots # ticking, unless like with '-l critical' which would be silent # unless error occurs @@ -587,9 +587,9 @@ def exec_command_line(): # pragma: no cover Formatter = logging.Formatter # Custom log format for the verbose tests runs - if opts.verbose: # pragma: no cover + if opts.verbose: stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) - else: # pragma: no cover + else: # just prefix with the space stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) From b3ed19b36adb17bfbd550bd0e32aac34e9ab8243 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:47:13 -0500 Subject: [PATCH 101/143] DOC: tune up to jail.conf.5 - some line breaks, typos etc --- man/jail.conf.5 | 50 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 7a31e8b1..865c689e 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -1,4 +1,4 @@ -.TH JAIL.CONF "5" "October 2013" "Fail2Ban" "Fail2Ban Configuration" +.TH JAIL.CONF "5" "November 2015" "Fail2Ban" "Fail2Ban Configuration" .SH NAME jail.conf \- configuration for the fail2ban server .SH SYNOPSIS @@ -89,8 +89,8 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. -Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g. .RS .nf @@ -100,7 +100,7 @@ failregex = %(known/failregex)s .fi .RE -Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file, e.g. .RS # filter.d/test.conf: @@ -118,7 +118,7 @@ filter = test[test.method=POST, baduseragents="badagent|"] .fi .RE -Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. +Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X, '; ' can only be used on the first line due to an Python library bug. .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" @@ -130,34 +130,44 @@ The items that can be set are: verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG. Default: ERROR .TP .B logtarget -log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR . Only a single log target can be specified. +log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR +.br +Only a single log target can be specified. If you change logtarget from the default value and you are using logrotate -- also adjust or disable rotation in the corresponding configuration file (e.g. /etc/logrotate.d/fail2ban on Debian systems). .TP .B socket -socket filename. Default: /var/run/fail2ban/fail2ban.sock . +socket filename. Default: /var/run/fail2ban/fail2ban.sock +.br This is used for communication with the fail2ban server daemon. Do not remove this file when Fail2ban is running. It will not be possible to communicate with the server afterwards. .TP .B pidfile -PID filename. Default: /var/run/fail2ban/fail2ban.pid. +PID filename. Default: /var/run/fail2ban/fail2ban.pid +.br This is used to store the process ID of the fail2ban server. .TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 +.br This defines where the persistent data for fail2ban is stored. This persistent data allows bans to be reinstated and continue reading log files from the last read position when fail2ban is restarted. A value of \fINone\fR disables this feature. .TP .B dbpurgeage Database purge age in seconds. Default: 86400 (24hours) +.br This sets the age at which bans should be purged from the database. .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP .B filter -name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. Only one filter can be specified. +name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. +.br +Only one filter can be specified. .TP .B logpath -filename(s) of the log files to be monitored, separated by new lines. Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. +filename(s) of the log files to be monitored, separated by new lines. +.br +Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. Optional space separated option 'tail' can be added to the end of the path to cause the log file to be read from the end, else default 'head' option reads file from the beginning @@ -167,13 +177,17 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. +.br +This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action -action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: +action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. +.br +Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS .RS @@ -187,7 +201,9 @@ Values can also be quoted (required when value includes a ","). More that one ac list of IPs not to ban. They can include a CIDR mask too. .TP .B ignorecommand -command that is executed to determine if the current candidate IP for banning should not be banned. IP will not be banned if command returns successfully (exit code 0). +command that is executed to determine if the current candidate IP for banning should not be banned. +.br +IP will not be banned if command returns successfully (exit code 0). Like ACTION FILES, tags like are can be included in the ignorecommand value and will be substituted before execution. Currently only is supported however more will be added later. .TP .B bantime @@ -200,7 +216,9 @@ time interval (in seconds) before the current time where failures will count tow number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP. .TP .B backend -backend to be used to detect changes in the logpath. It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. +backend to be used to detect changes in the logpath. +.br +It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. .TP .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. @@ -271,9 +289,9 @@ The maximum period of time in seconds that a command can executed, before being .RE Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should -return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). +return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed. From 9905396eb8bf0563a67ea9dc1d09d31a8af7e077 Mon Sep 17 00:00:00 2001 From: James Hartig Date: Wed, 11 Nov 2015 23:10:35 -0500 Subject: [PATCH 102/143] Added PartOf to service file so f2b restarts when deps do --- files/fail2ban.service | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service b/files/fail2ban.service index 6ebbacc0..e79faed1 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -2,6 +2,7 @@ Description=Fail2Ban Service Documentation=man:fail2ban(1) After=network.target iptables.service firewalld.service +PartOf=iptables.service firewalld.service [Service] Type=forking From fa59a6850ff5b093ed2c5928e5d86172bedc1aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandrs=20=C4=BBedovskis?= Date: Sun, 22 Nov 2015 12:01:15 +0200 Subject: [PATCH 103/143] Add 'Sender address rejected: Domain not found' Postfix failregex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aleksandrs Ļedovskis --- ChangeLog | 1 + config/filter.d/postfix.conf | 1 + fail2ban/tests/files/logs/postfix | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 20b708a8..15a3129d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fixed `fail2ban-regex` stops working on invalid (wrong encoded) character for python version < 3.x (gh-1248) * Use postfix_log logpath for postfix-rbl jail + * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex - New Features: * New interpolation feature for definition config readers - `` diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index a994d772..f6a8578b 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -16,6 +16,7 @@ failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7 ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index ee8720f8..4934a29e 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -23,3 +23,6 @@ Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after # failJSON: { "time": "2004-12-21T21:17:29", "match": true , "host": "93.184.216.34" } Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.1 Client host rejected: cannot find your hostname, [93.184.216.34]; from= to= proto=ESMTP helo= + +# failJSON: { "time": "2004-11-22T22:33:44", "match": true , "host": "1.2.3.4" } +Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.example.com[1.2.3.4]: 450 4.1.8 : Sender address rejected: Domain not found; from= to= proto=ESMTP helo=<1-2-3-4.example.com> From 5b88a84fe8f86817653ce1f4b5c0e48b80696878 Mon Sep 17 00:00:00 2001 From: sarneaud Date: Sat, 28 Nov 2015 15:03:09 +1100 Subject: [PATCH 104/143] Small fixes for Gentoo initd script These fixes are pretty pedantic, but they do simplify the script a little. * Checking the existence of a file/directory before creating/deleting it adds complexity and raciness. There are better options. * mkdir -p does the job of making sure a directory exists. (It only fails if there's a filesystem error or something.) * Likewise, rm -f doesn't fail if the file doesn't exist. * rm -r isn't neccessary because the socket shouldn't be a directory. (If it is for some reason, that should be an error.) --- files/gentoo-initd | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/files/gentoo-initd b/files/gentoo-initd index b56d4bdb..98c5edf9 100755 --- a/files/gentoo-initd +++ b/files/gentoo-initd @@ -30,14 +30,10 @@ depend() { start() { ebegin "Starting fail2ban" - if [ ! -d /var/run/fail2ban ]; then - mkdir /var/run/fail2ban || return 1 - fi - if [ -e /var/run/fail2ban/fail2ban.sock ]; then - # remove stalled sock file after system crash - # bug 347477 - rm -rf /var/run/fail2ban/fail2ban.sock || return 1 - fi + mkdir -p /var/run/fail2ban || return 1 + # remove stalled sock file after system crash + # bug 347477 + rm -f /var/run/fail2ban/fail2ban.sock || return 1 ${FAIL2BAN} start &> /dev/null eend $? "Failed to start fail2ban" } From 106c3eab9ab62828e6562ee1f0aa421e5af0312f Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Sun, 29 Nov 2015 15:56:56 +0000 Subject: [PATCH 105/143] Added filter and jail for murmur/mumble-server. --- config/filter.d/murmur.conf | 21 +++++++++++++++++++++ config/jail.conf | 9 +++++++++ 2 files changed, 30 insertions(+) create mode 100644 config/filter.d/murmur.conf diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf new file mode 100644 index 00000000..cc47f022 --- /dev/null +++ b/config/filter.d/murmur.conf @@ -0,0 +1,21 @@ +# Fail2Ban filter for murmur/mumble-server +# + +[INCLUDES] + +before = common.conf + + +[Definition] + +_daemon = murmurd + +failregex = Rejected connection from :\d+: Invalid server password$ + Rejected connection from :\d+: Wrong certificate or password for existing user$ + +ignoreregex = + + +# DEV Notes: +# +# Author: Ross Brown diff --git a/config/jail.conf b/config/jail.conf index c98392ba..c8dc6d9c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -821,3 +821,12 @@ returntype = DROP bantime = 3600 maxretry = 1 findtime = 1 + + +[murmur] +# AKA mumble-server +port = 64738 +filter = murmur +action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] + %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +logpath = /var/log/mumble-server/mumble-server.log From 4c837f033323fbbfd8bccd98a462e2139b3183e6 Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Sun, 29 Nov 2015 16:28:47 +0000 Subject: [PATCH 106/143] Added sample log file for 'murmur' filter. --- fail2ban/tests/files/logs/murmur | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 fail2ban/tests/files/logs/murmur diff --git a/fail2ban/tests/files/logs/murmur b/fail2ban/tests/files/logs/murmur new file mode 100644 index 00000000..0b738853 --- /dev/null +++ b/fail2ban/tests/files/logs/murmur @@ -0,0 +1,5 @@ +# failJSON: { "time": "2015-11-29T16:38:01", "match": true , "host": "192.168.0.1" } +2015-11-29 16:38:01.818 1 => <4:test(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password + +# failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" } +2015-11-29 17:18:20.962 1 => <8:test(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user From 3a179ec5d7121003683c16db77096247b0fe63f7 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Dec 2015 19:51:29 +0100 Subject: [PATCH 107/143] small code review: (much pretty) handling of filename as key - FileFilter contains (ordered) dict of files (not list), as discussed in gh-1265 --- ChangeLog | 2 ++ fail2ban/server/filter.py | 46 +++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/ChangeLog b/ChangeLog index 20b708a8..225a8a8b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) * Add *_backend options for services to allow distros to set the default backend per service, set default to systemd for Fedora as appropriate + * small improvement for better handling of many log files (gh-1265) + Thanks @kshetragia ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 65be0467..e0579c6e 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,6 +38,11 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from ..helpers import getLogger +try: + from collections import OrderedDict +except ImportError: + OrderedDict = dict + # Gets the instance of the logger. logSys = getLogger(__name__) @@ -552,7 +557,7 @@ class FileFilter(Filter): def __init__(self, jail, **kwargs): Filter.__init__(self, jail, **kwargs) ## The log file path. - self.__logPath = [] + self.__logs = OrderedDict() self.setLogEncoding("auto") ## @@ -561,7 +566,7 @@ class FileFilter(Filter): # @param path log file path def addLogPath(self, path, tail = False): - if self.containsLogPath(path): + if path in self.__logs: logSys.error(path + " already exists") else: container = FileContainer(path, self.getLogEncoding(), tail) @@ -570,7 +575,7 @@ class FileFilter(Filter): lastpos = db.addLog(self.jail, container) if lastpos and not tail: container.setPos(lastpos) - self.__logPath.append(container) + self.__logs[path] = container logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -585,15 +590,16 @@ class FileFilter(Filter): # @param path the log file to delete def delLogPath(self, path): - for log in self.__logPath: - if log.getFileName() == path: - self.__logPath.remove(log) - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) - logSys.info("Removed logfile = %s" % path) - self._delLogPath(path) - return + try: + log = self.__logs.pop(path) + except KeyError: + return + db = self.jail.database + if db is not None: + db.updateLog(self.jail, log) + logSys.info("Removed logfile = %s" % path) + self._delLogPath(path) + return def _delLogPath(self, path): # pragma: no cover - overwritten function # nothing to do by default @@ -606,7 +612,7 @@ class FileFilter(Filter): # @return log file path def getLogPath(self): - return self.__logPath + return self.__logs.values() ## # Check whether path is already monitored. @@ -615,10 +621,7 @@ class FileFilter(Filter): # @return True if the path is already monitored else False def containsLogPath(self, path): - for log in self.__logPath: - if log.getFileName() == path: - return True - return False + return path in self.__logs ## # Set the log file encoding @@ -629,7 +632,7 @@ class FileFilter(Filter): if encoding.lower() == "auto": encoding = locale.getpreferredencoding() codecs.lookup(encoding) # Raise LookupError if invalid codec - for log in self.getLogPath(): + for log in self.__logs.itervalues(): log.setEncoding(encoding) self.__encoding = encoding logSys.info("Set jail log file encoding to %s" % encoding) @@ -643,10 +646,7 @@ class FileFilter(Filter): return self.__encoding def getFileContainer(self, path): - for log in self.__logPath: - if log.getFileName() == path: - return log - return None + return self.__logs.get(path, None) ## # Gets all the failure in the log file. @@ -698,7 +698,7 @@ class FileFilter(Filter): """Status of Filter plus files being monitored. """ ret = super(FileFilter, self).status(flavor=flavor) - path = [m.getFileName() for m in self.getLogPath()] + path = self.__logs.keys() ret.append(("File list", path)) return ret From 6ce7522d3cebbc6a3adcd066d3189f7cff40536c Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Dec 2015 21:06:32 +0100 Subject: [PATCH 108/143] unordered (python 2.6) compatibility fix and coverage extended; --- fail2ban/tests/filtertestcase.py | 10 ++++++++++ fail2ban/tests/servertestcase.py | 20 ++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 3674a574..69b6e5b5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -859,6 +859,16 @@ class GetFailures(LogCaptureTestCase): self.filter.delLogPath(GetFailures.FILENAME_01) self.assertEqual(self.filter.getLogPath(),[]) + def testNoLogAdded(self): + self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) + self.assertTrue(self.filter.containsLogPath(GetFailures.FILENAME_01)) + self.filter.delLogPath(GetFailures.FILENAME_01) + self.assertFalse(self.filter.containsLogPath(GetFailures.FILENAME_01)) + # and unknown (safety and cover) + self.assertFalse(self.filter.containsLogPath('unknown.log')) + self.filter.delLogPath('unknown.log') + + def testGetFailures01(self, filename=None, failures=None): filename = filename or GetFailures.FILENAME_01 failures = failures or GetFailures.FAILURES_01 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 86ffdb46..07e10c7d 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -113,19 +113,15 @@ class TransmitterBase(unittest.TestCase): self.assertEqual( self.transm.proceed(["get", jail, cmd]), (0, [])) for n, value in enumerate(values): - self.assertEqual( - self.transm.proceed(["set", jail, cmdAdd, value]), - (0, values[:n+1])) - self.assertEqual( - self.transm.proceed(["get", jail, cmd]), - (0, values[:n+1])) + ret = self.transm.proceed(["set", jail, cmdAdd, value]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1]))) + ret = self.transm.proceed(["get", jail, cmd]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1]))) for n, value in enumerate(values): - self.assertEqual( - self.transm.proceed(["set", jail, cmdDel, value]), - (0, values[n+1:])) - self.assertEqual( - self.transm.proceed(["get", jail, cmd]), - (0, values[n+1:])) + ret = self.transm.proceed(["set", jail, cmdDel, value]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:]))) + ret = self.transm.proceed(["get", jail, cmd]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:]))) def jailAddDelRegexTest(self, cmd, inValues, outValues, jail): cmdAdd = "add" + cmd From dd9d1912e88d09910ae8173a70d76c58afd9d3f1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:49:47 -0500 Subject: [PATCH 109/143] RF: Filter.getLogPaths -> getLogs Since it returns log containers not paths per se --- fail2ban/server/filter.py | 8 ++++---- fail2ban/server/filtergamin.py | 4 ++-- fail2ban/server/filterpoll.py | 4 ++-- fail2ban/server/server.py | 2 +- fail2ban/tests/filtertestcase.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e0579c6e..ab4e4d5b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -565,7 +565,7 @@ class FileFilter(Filter): # # @param path log file path - def addLogPath(self, path, tail = False): + def addLogPath(self, path, tail=False): if path in self.__logs: logSys.error(path + " already exists") else: @@ -607,11 +607,11 @@ class FileFilter(Filter): pass ## - # Get the log file path + # Get the log containers # - # @return log file path + # @return log containers - def getLogPath(self): + def getLogs(self): return self.__logs.values() ## diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 1f51744b..e731a8e9 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -129,6 +129,6 @@ class FilterGamin(FileFilter): # Desallocates the resources used by Gamin. def __cleanup(self): - for path in self.getLogPath(): - self.monitor.stop_watch(path.getFileName()) + for log in self.getLogs(): + self.monitor.stop_watch(log.getFileName()) del self.monitor diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 25c3e119..d0b37775 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -88,10 +88,10 @@ class FilterPoll(FileFilter): while self.active: if logSys.getEffectiveLevel() <= 6: logSys.log(6, "Woke up idle=%s with %d files monitored", - self.idle, len(self.getLogPath())) + self.idle, len(self.getLogs())) if not self.idle: # Get file modification - for container in self.getLogPath(): + for container in self.getLogs(): filename = container.getFileName() if self.isModified(filename): self.getFailures(filename) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 6d19544d..3e371945 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -212,7 +212,7 @@ class Server: filter_ = self.__jails[name].filter if isinstance(filter_, FileFilter): return [m.getFileName() - for m in filter_.getLogPath()] + for m in filter_.getLogs()] else: # pragma: systemd no cover logSys.info("Jail %s is not a FileFilter instance" % name) return [] diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 69b6e5b5..39a1b352 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -853,11 +853,11 @@ class GetFailures(LogCaptureTestCase): def testTail(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) - self.assertEqual(self.filter.getLogPath()[-1].getPos(), 1653) - self.filter.getLogPath()[-1].close() - self.assertEqual(self.filter.getLogPath()[-1].readline(), "") + self.assertEqual(self.filter.getLogs()[-1].getPos(), 1653) + self.filter.getLogs()[-1].close() + self.assertEqual(self.filter.getLogs()[-1].readline(), "") self.filter.delLogPath(GetFailures.FILENAME_01) - self.assertEqual(self.filter.getLogPath(),[]) + self.assertEqual(self.filter.getLogs(), []) def testNoLogAdded(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) From 59da27b9f659badc2d28ada465eb575ebf4a651c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:53:28 -0500 Subject: [PATCH 110/143] ENH: add a check to testTail to assure correct test logic below it --- fail2ban/tests/filtertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 39a1b352..40879b66 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -852,6 +852,8 @@ class GetFailures(LogCaptureTestCase): LogCaptureTestCase.tearDown(self) def testTail(self): + # There must be no containters registered, otherwise [-1] indexing would be wrong + self.assertEqual(self.filter.getLogs(), []) self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) self.assertEqual(self.filter.getLogs()[-1].getPos(), 1653) self.filter.getLogs()[-1].close() From 48202f998d3ad672f8b56ae5e9d1854131d0a31c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:57:40 -0500 Subject: [PATCH 111/143] RF: prefer log over container in getLog and local variables Even though I have left FileContainer class name intact --- fail2ban/server/filter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ab4e4d5b..8f4f602a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -569,13 +569,13 @@ class FileFilter(Filter): if path in self.__logs: logSys.error(path + " already exists") else: - container = FileContainer(path, self.getLogEncoding(), tail) + log = FileContainer(path, self.getLogEncoding(), tail) db = self.jail.database if db is not None: - lastpos = db.addLog(self.jail, container) + lastpos = db.addLog(self.jail, log) if lastpos and not tail: - container.setPos(lastpos) - self.__logs[path] = container + log.setPos(lastpos) + self.__logs[path] = log logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -645,7 +645,7 @@ class FileFilter(Filter): def getLogEncoding(self): return self.__encoding - def getFileContainer(self, path): + def getLog(self, path): return self.__logs.get(path, None) ## @@ -656,13 +656,13 @@ class FileFilter(Filter): # is created and is added to the FailManager. def getFailures(self, filename): - container = self.getFileContainer(filename) - if container is None: + log = self.getLog(filename) + if log is None: logSys.error("Unable to get failures in " + filename) return False # Try to open log file. try: - has_content = container.open() + has_content = log.open() # see http://python.org/dev/peps/pep-3151/ except IOError, e: logSys.error("Unable to open %s" % filename) @@ -683,15 +683,15 @@ class FileFilter(Filter): # start reading tested to be empty container -- race condition # might occur leading at least to tests failures. while has_content: - line = container.readline() + line = log.readline() if not line or not self.active: # The jail reached the bottom or has been stopped break self.processLineAndAdd(line) - container.close() + log.close() db = self.jail.database if db is not None: - db.updateLog(self.jail, container) + db.updateLog(self.jail, log) return True def status(self, flavor="basic"): From 6d984717b5f743d836840d851b1d0814f3da842a Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 12 Dec 2015 15:46:45 +0100 Subject: [PATCH 112/143] ordered dict replaced with dict + change log entry fix # Conflicts: # fail2ban/server/filter.py --- ChangeLog | 5 +++-- fail2ban/server/filter.py | 7 +------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 225a8a8b..42fc0b38 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,8 +59,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) * Add *_backend options for services to allow distros to set the default backend per service, set default to systemd for Fedora as appropriate - * small improvement for better handling of many log files (gh-1265) - Thanks @kshetragia + * Performance improvements while monitoring large number of files (gh-1265). + Use associative array (dict) for monitored log files to speed up lookup + operations. Thanks @kshetragia ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8f4f602a..2b354f7a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,11 +38,6 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from ..helpers import getLogger -try: - from collections import OrderedDict -except ImportError: - OrderedDict = dict - # Gets the instance of the logger. logSys = getLogger(__name__) @@ -557,7 +552,7 @@ class FileFilter(Filter): def __init__(self, jail, **kwargs): Filter.__init__(self, jail, **kwargs) ## The log file path. - self.__logs = OrderedDict() + self.__logs = dict() self.setLogEncoding("auto") ## From 9ee08fed48388a8c81d2fb1f5c1fdf6c33b9b096 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Dec 2015 23:05:06 -0500 Subject: [PATCH 113/143] ENH: log at heavydebug level what actually we are matching for failregex --- fail2ban/server/filter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 2b354f7a..dae5c527 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -491,6 +491,7 @@ class Filter(JailThread): self.__lineBuffer = ( self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:] + logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer) # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): From 5d6cead99694efae8710ab12cd68a924a83d7396 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Dec 2015 23:21:04 -0500 Subject: [PATCH 114/143] ENH: sshd filter -- match new "maximum auth attempts exceeded" (Closes #1269) --- ChangeLog | 2 ++ config/filter.d/sshd.conf | 1 + fail2ban/tests/files/logs/sshd | 3 +++ 3 files changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 42fc0b38..90cdae59 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) - nginx-limit-req - ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) + * sshd filter got new failregex to match "maximum authentication + attempts exceeded" (introduced in openssh 6.8) - Enhancements: * Do not rotate empty log files diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 5fad2b32..180ac52a 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -33,6 +33,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?: on \S+ port \d+)?(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ + ^%(__prefix_line)s(error: )?maximum authentication attempts exceeded for .* from (?: port \d*)?(?: ssh\d*)? \[preauth\]$ ^%(__prefix_line)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 62204339..7baf4be7 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -148,6 +148,9 @@ Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" } Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] +# failJSON: { "time": "2004-11-23T21:50:37", "match": true , "host": "61.0.0.1", "desc": "New logline format as openssh 6.8 to replace prev multiline version" } +Nov 23 21:50:37 myhost sshd[21810]: error: maximum authentication attempts exceeded for root from 61.0.0.1 port 49940 ssh2 [preauth] + # failJSON: { "match": false } Apr 27 13:02:04 host sshd[29116]: User root not allowed because account is locked # failJSON: { "match": false } From ba535826a83136032c9e6f993804356f2890068e Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Tue, 15 Dec 2015 21:46:35 +0000 Subject: [PATCH 115/143] Updated ChangeLog to include new murmur filter. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 20b708a8..03bed0cd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) - nginx-limit-req - ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) + - murmur - ban hosts that repeatedly attempt to connect to + murmur/mumble-server with an invalid server password or certificate. - Enhancements: * Do not rotate empty log files From fd36b058cee39dac4524cef32a66931de017c6dd Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Tue, 15 Dec 2015 21:54:41 +0000 Subject: [PATCH 116/143] Changed usernames in sample log file for 'murmur' filter. --- fail2ban/tests/files/logs/murmur | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/murmur b/fail2ban/tests/files/logs/murmur index 0b738853..bc18b7ea 100644 --- a/fail2ban/tests/files/logs/murmur +++ b/fail2ban/tests/files/logs/murmur @@ -1,5 +1,5 @@ # failJSON: { "time": "2015-11-29T16:38:01", "match": true , "host": "192.168.0.1" } -2015-11-29 16:38:01.818 1 => <4:test(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password +2015-11-29 16:38:01.818 1 => <4:testUsernameOne(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password # failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" } -2015-11-29 17:18:20.962 1 => <8:test(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user +2015-11-29 17:18:20.962 1 => <8:testUsernameTwo(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user From ead2d509dc0101807f0cad53e896963a5498e47b Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Thu, 17 Dec 2015 17:45:24 +0000 Subject: [PATCH 117/143] Updated 'murmur' filter to use new double-anchored regex based on @yarikoptic's suggestions. --- config/filter.d/murmur.conf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf index cc47f022..3775a9d2 100644 --- a/config/filter.d/murmur.conf +++ b/config/filter.d/murmur.conf @@ -10,8 +10,15 @@ before = common.conf _daemon = murmurd -failregex = Rejected connection from :\d+: Invalid server password$ - Rejected connection from :\d+: Wrong certificate or password for existing user$ +# N.B. If you allow users to have usernames that include the '>' character you +# should change this to match the regex assigned to the 'username' +# variable in your server config file (murmur.ini / mumble-server.ini). +_usernameregex = [^>]+ + +_prefix = [\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from :\d+: + +failregex = ^%(_prefix)s Invalid server password$ + ^%(_prefix)s Wrong certificate or password for existing user$ ignoreregex = From 16aa2fa13e387b8eb539349192879fb9af9170b3 Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Thu, 17 Dec 2015 17:57:45 +0000 Subject: [PATCH 118/143] Updated ChangeLog to include new murmur jail. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 03bed0cd..20ca6def 100644 --- a/ChangeLog +++ b/ChangeLog @@ -43,6 +43,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released request processing rate (ngx_http_limit_req_module) - murmur - ban hosts that repeatedly attempt to connect to murmur/mumble-server with an invalid server password or certificate. + * New jails: + - murmur - bans TCP and UDP from the bad host on the default murmur port. - Enhancements: * Do not rotate empty log files From a3e0cc9830969f09442fc79d8c8be6353791ec49 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 14 May 2014 16:01:35 +0200 Subject: [PATCH 119/143] introduced string to seconds (str2seconds) for configuration entries with time; todo: expands it for all time config entries; --- fail2ban/server/mytime.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 166f4796..0a7e6a79 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import datetime +import re import time @@ -96,3 +97,32 @@ class MyTime: return time.localtime(x) else: return time.localtime(MyTime.myTime) + + ## + # Wraps string expression like "1h 2m 3s" into number contains seconds (3723). + # The string expression will be evaluated as mathematical expression, spaces between each groups + # will be wrapped to "+" operand (only if any operand does not specified between). + # Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month + # are "mo" or "mon" (like %b by date formating). + # Ex: 1hour+30min = 5400 + # 0d 1h 30m = 5400 + # 1year-6mo = 15778800 + # 6 months = 15778800 + # warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also: + # >>>> float(Test.str2seconds("1month")) / 60 / 60 / 24 + # 30.4375 + # >>>> float(Test.str2seconds("1year")) / 60 / 60 / 24 + # 365.25 + # + # @returns number (calculated seconds from expression "val") + + #@staticmethod + def str2seconds(val): + for rexp, rpl in ( + (r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), + (r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60), + ): + val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val) + val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val); + return eval(val) + str2seconds = staticmethod(str2seconds) From 3cfdc5ecb36796ca2c5733ed8e20668821c442ee Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jun 2014 12:31:55 +0200 Subject: [PATCH 120/143] test cases extended; code review --- fail2ban/tests/dummyjail.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 9b784f77..6be571d3 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -24,18 +24,19 @@ __license__ = "GPL" from threading import Lock +from ..server.jail import Jail from ..server.actions import Actions -class DummyJail(object): +class DummyJail(Jail, object): """A simple 'jail' to suck in all the tickets generated by Filter's """ - def __init__(self): + def __init__(self, backend=None): self.lock = Lock() self.queue = [] - self.idle = False - self.database = None - self.actions = Actions(self) + super(DummyJail, self).__init__(name='DummyJail', backend=backend) + self.__db = None + self.__actions = Actions(self) def __len__(self): try: @@ -64,3 +65,26 @@ class DummyJail(object): @property def name(self): return "DummyJail #%s with %d tickets" % (id(self), len(self)) + + @property + def idle(self): + return False; + + @idle.setter + def idle(self, value): + pass + + @property + def database(self): + return self.__db; + + @database.setter + def database(self, value): + self.__db = value; + + @property + def actions(self): + return self.__actions; + + def is_alive(self): + return True; From 42199957d02de7dd2abd5c651638b635a0f7860b Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 16 Sep 2014 13:50:32 +0200 Subject: [PATCH 121/143] prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm (see issue #795): now polling backend only (currently not implemented for gamin and pyinotify backends); python3/pypy compatibility fix + removing obsolete code --- fail2ban/server/filter.py | 90 +++++++++++++++++++++++++++++++- fail2ban/server/filterpoll.py | 7 ++- fail2ban/tests/filtertestcase.py | 9 ++++ 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index dae5c527..2dfeb174 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -24,6 +24,7 @@ __license__ = "GPL" import codecs import fcntl import locale +import logging import os import re import sys @@ -190,6 +191,7 @@ class Filter(JailThread): # @param value the time def setFindTime(self, value): + value = MyTime.str2seconds(value) self.__findTime = value self.failManager.setMaxTime(value) logSys.info("Set findtime = %s" % value) @@ -651,7 +653,7 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename): + def getFailures(self, filename, startTime=None): log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in " + filename) @@ -673,6 +675,11 @@ class FileFilter(Filter): logSys.exception(e) return False + # prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm: + if log.getPos() == 0 and startTime is not None: + # startTime = MyTime.time() - self.getFindTime() + self.seekToTime(log, startTime) + # yoh: has_content is just a bool, so do not expect it to # change -- loop is exited upon break, and is not entered at # all if upon container opening that one was empty. If we @@ -690,6 +697,74 @@ class FileFilter(Filter): db.updateLog(self.jail, log) return True + ## + # Seeks to line with date (search using half-interval search algorithm), to start polling from it + # + + def seekToTime(self, container, date): + fs = container.getFileSize() + if logSys.getEffectiveLevel() <= logging.DEBUG: + logSys.debug("Seek to find time %s (%s), file size %s", date, + datetime.datetime.fromtimestamp(date).strftime("%Y-%m-%d %H:%M:%S"), fs) + date -= 0.009 + minp = 0 + maxp = fs + lastpos = 0 + lastFew = 0 + lastTime = None + cntr = 0 + unixTime = None + lasti = 0 + movecntr = 3 + while maxp > minp: + i = int(minp + (maxp - minp) / 2) + pos = container.seek(i) + cntr += 1 + # within next 5 lines try to find any legal datetime: + lncntr = 5; + dateTimeMatch = None + llen = 0 + i = pos + while True: + line = container.readline() + if not line: + break + llen += len(line) + l = line.rstrip('\r\n') + timeMatch = self.dateDetector.matchTime(l) + if timeMatch: + dateTimeMatch = self.dateDetector.getTime(l[timeMatch.start():timeMatch.end()]) + if not dateTimeMatch and lncntr: + lncntr -= 1 + continue + break + # if we can't move (position not changed) + if i + llen == lasti: + movecntr -= 1 + if movecntr <= 0: + break + lasti = i + llen; + # not found at this step - stop searching + if not dateTimeMatch: + break + unixTime = dateTimeMatch[0] + if unixTime >= date: + maxp = i + else: + minp = i + llen + lastFew = pos; + lastTime = unixTime + lastpos = pos + # if found position have a time greater as given - use smallest time we have found + if unixTime is None or unixTime > date: + unixTime = lastTime + lastpos = container.seek(lastFew, False) + else: + lastpos = container.seek(lastpos, False) + if logSys.getEffectiveLevel() <= logging.DEBUG: + logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, + (datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr) + def status(self, flavor="basic"): """Status of Filter plus files being monitored. """ @@ -742,6 +817,9 @@ class FileContainer: def getFileName(self): return self.__filename + def getFileSize(self): + return os.path.getsize(self.__filename); + def setEncoding(self, encoding): codecs.lookup(encoding) # Raises LookupError if invalid self.__encoding = encoding @@ -788,6 +866,16 @@ class FileContainer: self.__handler.seek(self.__pos) return True + def seek(self, offs, endLine=True): + h = self.__handler + # seek to given position + h.seek(offs, 0) + # goto end of next line + if endLine: + h.readline() + # get current real position + return h.tell() + @staticmethod def decode_line(filename, enc, line): try: diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index d0b37775..0e99455e 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -57,6 +57,7 @@ class FilterPoll(FileFilter): ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() + self.__initial = dict() logSys.debug("Created FilterPoll") ## @@ -94,7 +95,11 @@ class FilterPoll(FileFilter): for container in self.getLogs(): filename = container.getFileName() if self.isModified(filename): - self.getFailures(filename) + # set start time as now - find time for first usage only (prevent performance bug with polling of big files) + self.getFailures(filename, + (MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None + ) + self.__initial[filename] = True self.__modified = True if self.__modified: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 40879b66..c7dba5ce 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -912,6 +912,15 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_seek(self): + # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... + output = ('203.162.223.135', 5, 1124013544.0) + + self.filter.addLogPath(GetFailures.FILENAME_03) + self.filter.addFailRegex("error,relay=,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03, output[2] - 4*60 + 1) + _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures04(self): output = [('212.41.96.186', 4, 1124013600.0), ('212.41.96.185', 4, 1124017198.0)] From 410f9d7c101b5987071fcd829d6eeac3bba0b505 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Sep 2014 13:21:37 +0200 Subject: [PATCH 122/143] filter, datedetector, datetemplate: performance optimizing of combination datedetector.matchTime/getTime2, because early getTime search a template and call template.matchTime again (so the date parsing was really executed twice, now just once); debug logging optimized; added info line log "Start Fail2ban ..." after changed logging target; --- fail2ban/server/action.py | 10 +++-- fail2ban/server/datedetector.py | 56 +++++++++++++++++++++----- fail2ban/server/datetemplate.py | 17 ++++---- fail2ban/server/filter.py | 16 ++++---- fail2ban/server/server.py | 3 +- fail2ban/tests/datedetectortestcase.py | 15 +++++++ 6 files changed, 89 insertions(+), 28 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index de0c8efc..85bdd166 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -592,10 +592,12 @@ class CommandAction(ActionBase): std_level = retcode == 0 and logging.DEBUG or logging.ERROR if std_level >= logSys.getEffectiveLevel(): - stdout.seek(0) - logSys.log(std_level, "%s -- stdout: %r" % (realCmd, stdout.read())) - stderr.seek(0) - logSys.log(std_level, "%s -- stderr: %r" % (realCmd, stderr.read())) + stdout.seek(0); msg = stdout.read() + if msg != '': + logSys.log(std_level, "%s -- stdout: %r", realCmd, msg) + stderr.seek(0); msg = stderr.read() + if msg != '': + logSys.log(std_level, "%s -- stderr: %r", realCmd, msg) stdout.close() stderr.close() diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index a4839f5c..dbfb9e5d 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -29,6 +29,8 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = 6 + class DateDetector(object): """Manages one or more date templates to find a date within a log line. @@ -147,7 +149,7 @@ class DateDetector(object): Returns ------- - re.MatchObject + re.MatchObject, DateTemplate The regex match returned from the first successfully matched template. """ @@ -156,10 +158,11 @@ class DateDetector(object): for template in self.__templates: match = template.matchDate(line) if not match is None: - logSys.debug("Matched time template %s" % template.name) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Matched time template %s", template.name) template.hits += 1 - return match - return None + return (match, template) + return (None, None) finally: self.__lock.release() @@ -178,7 +181,7 @@ class DateDetector(object): ------- float The Unix timestamp returned from the first successfully matched - template. + template or None if not found. """ self.__lock.acquire() try: @@ -187,8 +190,9 @@ class DateDetector(object): date = template.getDate(line) if date is None: continue - logSys.debug("Got time %f for \"%r\" using template %s" % - (date[0], date[1].group(), template.name)) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Got time %f for \"%r\" using template %s", + date[0], date[1].group(), template.name) return date except ValueError: pass @@ -196,6 +200,38 @@ class DateDetector(object): finally: self.__lock.release() + def getTime2(self, line, timeMatch = None): + """Attempts to return the date on a log line using given template. + + This uses the templates' `getDate` method in an attempt to find + a date. + Method 'getTime2' is a little bit faster as 'getTime' if template was specified (cause works without locking and without cycle) + + Parameters + ---------- + line : str + Line which is searched by the date templates. + timeMatch (timeMatch, template) : (Match, DateTemplate) + Time match and template previously returned from matchTime + + Returns + ------- + float + The Unix timestamp returned from the first successfully matched + template or None if not found. + """ + date = None + if timeMatch: + template = timeMatch[1] + if template is not None: + date = template.getDate(line, timeMatch[0]) + if date is not None: + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Got time(2) %f for \"%r\" using template %s", + date[0], date[1].group(), template.name) + return date + return self.getTime(line) + def sortTemplate(self): """Sort the date templates by number of hits @@ -206,9 +242,11 @@ class DateDetector(object): """ self.__lock.acquire() try: - logSys.debug("Sorting the template list") + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Sorting the template list") self.__templates.sort(key=lambda x: x.hits, reverse=True) t = self.__templates[0] - logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits)) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Winning template: %s with %d hits", t.name, t.hits) finally: self.__lock.release() diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index bcd17ec1..e1f6b3de 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -98,7 +98,7 @@ class DateTemplate(object): return dateMatch @abstractmethod - def getDate(self, line): + def getDate(self, line, dateMatch=None): """Abstract method, which should return the date for a log line This should return the date for a log line, typically taking the @@ -134,7 +134,7 @@ class DateEpoch(DateTemplate): DateTemplate.__init__(self) self.regex = "(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))" - def getDate(self, line): + def getDate(self, line, dateMatch=None): """Method to return the date for a log line. Parameters @@ -148,7 +148,8 @@ class DateEpoch(DateTemplate): Tuple containing a Unix timestamp, and the string of the date which was matched and in turned used to calculated the timestamp. """ - dateMatch = self.matchDate(line) + if not dateMatch: + dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch return (float(dateMatch.group()), dateMatch) @@ -212,7 +213,7 @@ class DatePatternRegex(DateTemplate): def name(self, value): raise NotImplementedError("Name derived from pattern") - def getDate(self, line): + def getDate(self, line, dateMatch=None): """Method to return the date for a log line. This uses a custom version of strptime, using the named groups @@ -229,7 +230,8 @@ class DatePatternRegex(DateTemplate): Tuple containing a Unix timestamp, and the string of the date which was matched and in turned used to calculated the timestamp. """ - dateMatch = self.matchDate(line) + if not dateMatch: + dateMatch = self.matchDate(line) if dateMatch: groupdict = dict( (key, value) @@ -253,7 +255,7 @@ class DateTai64n(DateTemplate): # yoh: we should not add an additional front anchor self.setRegex("@[0-9a-f]{24}", wordBegin=False) - def getDate(self, line): + def getDate(self, line, dateMatch=None): """Method to return the date for a log line. Parameters @@ -267,7 +269,8 @@ class DateTai64n(DateTemplate): Tuple containing a Unix timestamp, and the string of the date which was matched and in turned used to calculated the timestamp. """ - dateMatch = self.matchDate(line) + if not dateMatch: + dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch value = dateMatch.group() diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 2dfeb174..296ff101 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -406,14 +406,16 @@ class Filter(JailThread): l = line.rstrip('\r\n') logSys.log(7, "Working on line %r", line) - timeMatch = self.dateDetector.matchTime(l) + (timeMatch, template) = self.dateDetector.matchTime(l) if timeMatch: tupleLine = ( l[:timeMatch.start()], l[timeMatch.start():timeMatch.end()], - l[timeMatch.end():]) + l[timeMatch.end():], + (timeMatch, template) + ) else: - tupleLine = (l, "", "") + tupleLine = (l, "", "", None) return "".join(tupleLine[::2]), self.findFailure( tupleLine, date, returnRawHost, checkAllRegex) @@ -475,7 +477,7 @@ class Filter(JailThread): self.__lastDate = date elif timeText: - dateTimeMatch = self.dateDetector.getTime(timeText) + dateTimeMatch = self.dateDetector.getTime2(timeText, tupleLine[3]) if dateTimeMatch is None: logSys.error("findFailure failed to parse timeText: " + timeText) @@ -492,7 +494,7 @@ class Filter(JailThread): date = self.__lastDate self.__lineBuffer = ( - self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:] + self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer) # Iterates over all the regular expressions. @@ -731,9 +733,9 @@ class FileFilter(Filter): break llen += len(line) l = line.rstrip('\r\n') - timeMatch = self.dateDetector.matchTime(l) + (timeMatch, template) = self.dateDetector.matchTime(l) if timeMatch: - dateTimeMatch = self.dateDetector.getTime(l[timeMatch.start():timeMatch.end()]) + dateTimeMatch = self.dateDetector.getTime2(l[timeMatch.start():timeMatch.end()], (timeMatch, template)) if not dateTimeMatch and lncntr: lncntr -= 1 continue diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 3e371945..9836b61d 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -81,7 +81,7 @@ class Server: self.flushLogs() def start(self, sock, pidfile, force = False): - logSys.info("Starting Fail2ban v" + version.version) + logSys.info("Starting Fail2ban v%s", version.version) # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) @@ -443,6 +443,7 @@ class Server: logger.addHandler(hdlr) # Does not display this message at startup. if not self.__logTarget is None: + logSys.info("Start Fail2ban v%s", version.version) logSys.info( "Changed logging target to %s for Fail2ban v%s" % ((target diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index facfda6d..095b706a 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -106,6 +106,7 @@ class DateDetectorTest(unittest.TestCase): (not anchored, "bogus-prefix ")): log = prefix + sdate + "[sshd] error: PAM: Authentication failure" + # with getTime: logtime = self.__datedetector.getTime(log) if should_match: self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log)) @@ -118,6 +119,20 @@ class DateDetectorTest(unittest.TestCase): self.assertEqual(logMatch.group(), sdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) + # with matchTime and getTime2 (this combination used in filter) : + matchTime = self.__datedetector.matchTime(log) + logtime = self.__datedetector.getTime2(log, matchTime) + if should_match: + self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log)) + ( logUnix, logMatch ) = logtime + self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix)) + if sdate.startswith('audit('): + # yes, special case, the group only matches the number + self.assertEqual(logMatch.group(), '1106513999.000') + else: + self.assertEqual(logMatch.group(), sdate) + else: + self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) def testStableSortTemplate(self): old_names = [x.name for x in self.__datedetector.templates] From dad4234beb325a1d9836c525575956d768938208 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 25 Sep 2014 18:29:10 +0200 Subject: [PATCH 123/143] The tricky bug fixed - last position of log file will be never retrieved (#795): addJail (executed before addLog) early uses a "INSERT OR REPLACE" statement to update "enabled" to 1 (and add jail the first time used at once), but this syntax in sqlite always deletes an entry (cause of constraint) and inserts it again, so because of CASCADE all log entries with this jail will be also deleted from logs table. --- fail2ban/server/database.py | 6 +++++- fail2ban/server/filter.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 7de87554..fcc56f73 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -274,8 +274,12 @@ class Fail2BanDb(object): Jail to be added to the database. """ cur.execute( - "INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)", + "INSERT OR IGNORE INTO jails(name, enabled) VALUES(?, 1)", (jail.name,)) + if cur.rowcount <= 0: + cur.execute( + "UPDATE jails SET enabled = 1 WHERE name = ? AND enabled != 1", + (jail.name,)) @commitandrollback def delJail(self, cur, jail): diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 296ff101..ee0e2a1d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -576,7 +576,7 @@ class FileFilter(Filter): if lastpos and not tail: log.setPos(lastpos) self.__logs[path] = log - logSys.info("Added logfile = %s" % path) + logSys.info("Added logfile = %s (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) self._addLogPath(path) # backend specific def _addLogPath(self, path): From d22b2498d4dd2ba3f4ea45260335783648eb62cf Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Oct 2014 01:32:04 +0200 Subject: [PATCH 124/143] normalizing time config entries: use time abbreviation (str2seconds) for all time options such 'dbpurgeage', 'bantime', 'findtime', ex.: default '1d' instead '86400'; code review and test case extended; --- config/fail2ban.conf | 2 +- config/jail.conf | 14 +++++++------- fail2ban/client/fail2banreader.py | 2 +- fail2ban/client/jailreader.py | 6 +++--- fail2ban/server/database.py | 2 +- fail2ban/server/mytime.py | 6 ++++-- fail2ban/server/transmitter.py | 4 ++-- fail2ban/tests/clientreadertestcase.py | 2 +- fail2ban/tests/databasetestcase.py | 9 +++++++++ fail2ban/tests/servertestcase.py | 2 ++ 10 files changed, 31 insertions(+), 18 deletions(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index 89e2538e..7c001155 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -66,4 +66,4 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Options: dbpurgeage # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) -dbpurgeage = 86400 +dbpurgeage = 1d diff --git a/config/jail.conf b/config/jail.conf index c8dc6d9c..21b97798 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -18,7 +18,7 @@ # See man 5 jail.conf for details. # # [DEFAULT] -# bantime = 3600 +# bantime = 1h # # [sshd] # enabled = true @@ -56,11 +56,11 @@ ignoreip = 127.0.0.1/8 ignorecommand = # "bantime" is the number of seconds that a host is banned. -bantime = 600 +bantime = 10m # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 600 +findtime = 10m # "maxretry" is the number of failures before a host get banned. maxretry = 5 @@ -258,7 +258,7 @@ logpath = %(apache_error_log)s # for email addresses. The mail outputs are buffered. port = http,https logpath = %(apache_access_log)s -bantime = 172800 +bantime = 48h maxretry = 1 @@ -738,8 +738,8 @@ maxretry = 5 logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s -bantime = 604800 ; 1 week -findtime = 86400 ; 1 day +bantime = 1w +findtime = 1d maxretry = 5 @@ -818,7 +818,7 @@ filter = apache-pass logpath = %(apache_access_log)s blocktype = RETURN returntype = DROP -bantime = 3600 +bantime = 1h maxretry = 1 findtime = 1 diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 709f4b5d..c55f65ea 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -49,7 +49,7 @@ class Fail2banReader(ConfigReader): ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], - ["int", "dbpurgeage", 86400]] + ["string", "dbpurgeage", "1d"]] self.__opts = ConfigReader.getOptions(self, "Definition", opts) def convert(self): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 54ac59fa..46f910e3 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -97,9 +97,9 @@ class JailReader(ConfigReader): ["string", "logpath", None], ["string", "logencoding", None], ["string", "backend", "auto"], - ["int", "maxretry", None], - ["int", "findtime", None], - ["int", "bantime", None], + ["int", "maxretry", None], + ["string", "findtime", None], + ["string", "bantime", None], ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index fcc56f73..3cfaa5a6 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -218,7 +218,7 @@ class Fail2BanDb(object): @purgeage.setter def purgeage(self, value): - self._purgeAge = int(value) + self._purgeAge = MyTime.str2seconds(value) @commitandrollback def createDb(self, cur): diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 0a7e6a79..47509f19 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -116,8 +116,11 @@ class MyTime: # # @returns number (calculated seconds from expression "val") - #@staticmethod + @staticmethod def str2seconds(val): + # replace together standing abbreviations, example '1d12h' -> '1d 12h': + val = re.sub(r"(?i)(?<=[a-z])(\d)", r" \1", val) + # replace abbreviation with expression: for rexp, rpl in ( (r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), (r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60), @@ -125,4 +128,3 @@ class MyTime: val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val) val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val); return eval(val) - str2seconds = staticmethod(str2seconds) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 0d9f0fe4..eb5b7713 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -216,7 +216,7 @@ class Transmitter: return self.__server.getUseDns(name) elif command[1] == "findtime": value = command[2] - self.__server.setFindTime(name, int(value)) + self.__server.setFindTime(name, value) return self.__server.getFindTime(name) elif command[1] == "datepattern": value = command[2] @@ -233,7 +233,7 @@ class Transmitter: # command elif command[1] == "bantime": value = command[2] - self.__server.setBanTime(name, int(value)) + self.__server.setBanTime(name, value) return self.__server.getBanTime(name) elif command[1] == "banip": value = command[2] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d19090be..d0c37a3e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -668,7 +668,7 @@ class JailsReaderTest(LogCaptureTestCase): self.assertEqual(sorted(commands), [['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'], - ['set', 'dbpurgeage', 86400], + ['set', 'dbpurgeage', '1d'], ['set', 'loglevel', "INFO"], ['set', 'logtarget', '/var/log/fail2ban.log'], ['set', 'syslogsocket', 'auto']]) diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 3d156eda..083be2b2 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -70,6 +70,15 @@ class DatabaseTest(LogCaptureTestCase): return self.assertEqual(self.dbFilename, self.db.filename) + def testPurgeAge(self): + if Fail2BanDb is None: # pragma: no cover + return + self.assertEqual(self.db.purgeage, 86400) + self.db.purgeage = '1y6mon15d5h30m' + self.assertEqual(self.db.purgeage, 48652200) + self.db.purgeage = '2y 12mon 30d 10h 60m' + self.assertEqual(self.db.purgeage, 48652200*2) + def testCreateInvalidPath(self): if Fail2BanDb is None: # pragma: no cover return diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 07e10c7d..a7d7adef 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -259,6 +259,7 @@ class Transmitter(TransmitterBase): def testJailFindTime(self): self.setGetTest("findtime", "120", 120, jail=self.jailName) self.setGetTest("findtime", "60", 60, jail=self.jailName) + self.setGetTest("findtime", "30m", 30*60, jail=self.jailName) self.setGetTest("findtime", "-60", -60, jail=self.jailName) self.setGetTestNOK("findtime", "Dog", jail=self.jailName) @@ -266,6 +267,7 @@ class Transmitter(TransmitterBase): self.setGetTest("bantime", "600", 600, jail=self.jailName) self.setGetTest("bantime", "50", 50, jail=self.jailName) self.setGetTest("bantime", "-50", -50, jail=self.jailName) + self.setGetTest("bantime", "15d 5h 30m", 1315800, jail=self.jailName) self.setGetTestNOK("bantime", "Cat", jail=self.jailName) def testDatePattern(self): From db5ee0842a56a5db0e933e0125b0d0ab4773c86a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Oct 2014 21:57:16 +0100 Subject: [PATCH 125/143] increase code coverage --- fail2ban/server/datedetector.py | 2 +- fail2ban/server/jail.py | 6 ++++-- fail2ban/tests/datedetectortestcase.py | 13 +++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index dbfb9e5d..4bdcefd2 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -194,7 +194,7 @@ class DateDetector(object): logSys.log(logLevel, "Got time %f for \"%r\" using template %s", date[0], date[1].group(), template.name) return date - except ValueError: + except ValueError: # pragma: no cover pass return None finally: diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index a866cb51..c5f2e9cd 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -76,7 +76,8 @@ class Jail: self.__queue = Queue.Queue() self.__filter = None logSys.info("Creating new jail '%s'" % self.name) - self._setBackend(backend) + if backend is not None: + self._setBackend(backend) def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self.name) @@ -106,11 +107,12 @@ class Jail: logSys.info("Initiated %r backend" % b) self.__actions = Actions(self) return # we are done - except ImportError, e: + except ImportError, e: # pragma: no cover # Log debug if auto, but error if specific logSys.log( logging.DEBUG if backend == "auto" else logging.ERROR, "Backend %r failed to initialize due to %s" % (b, e)) + # pragma: no cover # log error since runtime error message isn't printed, INVALID COMMAND logSys.error( "Failed to initialize any backend for Jail %r" % self.name) diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 095b706a..9a7d3e78 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -29,20 +29,29 @@ import time import datetime from ..server.datedetector import DateDetector +from ..server import datedetector from ..server.datetemplate import DateTemplate -from .utils import setUpMyTime, tearDownMyTime +from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase +from ..helpers import getLogger + +logSys = getLogger("fail2ban") -class DateDetectorTest(unittest.TestCase): +class DateDetectorTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) + self.__old_eff_level = datedetector.logLevel + datedetector.logLevel = logSys.getEffectiveLevel() setUpMyTime() self.__datedetector = DateDetector() self.__datedetector.addDefaultTemplate() def tearDown(self): """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + datedetector.logLevel = self.__old_eff_level tearDownMyTime() def testGetEpochTime(self): From d23d103f967440942ff598e7543153a54be3f738 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 2 Dec 2014 00:56:20 +0100 Subject: [PATCH 126/143] code review and few new test cases --- fail2ban/server/actions.py | 1 + fail2ban/server/mytime.py | 42 ++++++++++++++++++---------------- fail2ban/tests/misctestcase.py | 18 +++++++++++++++ fail2ban/tests/utils.py | 3 ++- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index b4612f8c..c00dc7fc 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -167,6 +167,7 @@ class Actions(JailThread, Mapping): # @param value the time def setBanTime(self, value): + value = MyTime.str2seconds(value) self.__banManager.setBanTime(value) logSys.info("Set banTime = %s" % value) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 47509f19..329e2465 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -98,32 +98,34 @@ class MyTime: else: return time.localtime(MyTime.myTime) - ## - # Wraps string expression like "1h 2m 3s" into number contains seconds (3723). - # The string expression will be evaluated as mathematical expression, spaces between each groups - # will be wrapped to "+" operand (only if any operand does not specified between). - # Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month - # are "mo" or "mon" (like %b by date formating). - # Ex: 1hour+30min = 5400 - # 0d 1h 30m = 5400 - # 1year-6mo = 15778800 - # 6 months = 15778800 - # warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also: - # >>>> float(Test.str2seconds("1month")) / 60 / 60 / 24 - # 30.4375 - # >>>> float(Test.str2seconds("1year")) / 60 / 60 / 24 - # 365.25 - # - # @returns number (calculated seconds from expression "val") - @staticmethod def str2seconds(val): + """Wraps string expression like "1h 2m 3s" into number contains seconds (3723). + The string expression will be evaluated as mathematical expression, spaces between each groups + will be wrapped to "+" operand (only if any operand does not specified between). + Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month + are "mo" or "mon". + Ex: 1hour+30min = 5400 + 0d 1h 30m = 5400 + 1year-6mo = 15778800 + 6 months = 15778800 + warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also: + >>>> float(str2seconds("1month")) / 60 / 60 / 24 + 30.4375 + >>>> float(str2seconds("1year")) / 60 / 60 / 24 + 365.25 + + @returns number (calculated seconds from expression "val") + """ + if isinstance(val, (int, long, float, complex)): + return val # replace together standing abbreviations, example '1d12h' -> '1d 12h': val = re.sub(r"(?i)(?<=[a-z])(\d)", r" \1", val) # replace abbreviation with expression: for rexp, rpl in ( - (r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), - (r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60), + (r"days?|da|dd?", 24*60*60), (r"weeks?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), + (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), + (r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|hou?|hh?", 60*60), ): val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val) val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val); diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index e28ce422..a2310ec5 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -35,6 +35,7 @@ from StringIO import StringIO from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger from ..helpers import splitcommaspace from ..server.datetemplate import DatePatternRegex +from ..server.mytime import MyTime class HelpersTest(unittest.TestCase): @@ -229,3 +230,20 @@ class CustomDateFormatsTest(unittest.TestCase): self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) + +class MyTimeTest(unittest.TestCase): + + def testStr2Seconds(self): + # several formats / write styles: + str2sec = MyTime.str2seconds + self.assertEqual(str2sec('1y6mo30w15d12h35m25s'), 66821725) + self.assertEqual(str2sec('2yy 3mo 4ww 10dd 5hh 30mm 20ss'), 74307620) + self.assertEqual(str2sec('2 years 3 months 4 weeks 10 days 5 hours 30 minutes 20 seconds'), 74307620) + self.assertEqual(str2sec('1 year + 1 month - 1 week + 1 day'), 33669000) + self.assertEqual(str2sec('2 * 0.5 yea + 1*1 mon - 3*1/3 wee + 2/2 day - (2*12 hou 3*20 min 80 sec) '), 33578920.0) + self.assertEqual(str2sec('2*.5y+1*1mo-3*1/3w+2/2d-(2*12h3*20m80s) '), 33578920.0) + self.assertEqual(str2sec('1ye -2mo -3we -4da -5ho -6mi -7se'), 24119633) + # month and year in days : + self.assertEqual(float(str2sec("1 month")) / 60 / 60 / 24, 30.4375) + self.assertEqual(float(str2sec("1 year")) / 60 / 60 / 24, 365.25) + diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8172e7ec..dff3faae 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -133,6 +133,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(misctestcase.SetupTest)) tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest)) tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest)) + tests.addTest(unittest.makeSuite(misctestcase.MyTimeTest)) # Database tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest)) @@ -300,4 +301,4 @@ else: kernel32.CloseHandle(process) return True else: - return False \ No newline at end of file + return False From 3540619a736644180c603dd324c8242d0f4e9df4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 2 Dec 2014 11:57:43 +0100 Subject: [PATCH 127/143] code review, test case extended; --- fail2ban/server/filter.py | 23 +++++++------ fail2ban/tests/filtertestcase.py | 58 ++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ee0e2a1d..621f826b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -673,14 +673,19 @@ class FileFilter(Filter): logSys.exception(e) return False except OSError, e: # pragma: no cover - Requires implemention error in FileContainer to generate - logSys.error("Internal errror in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues") + logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues") logSys.exception(e) return False # prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm: if log.getPos() == 0 and startTime is not None: - # startTime = MyTime.time() - self.getFindTime() - self.seekToTime(log, startTime) + try: + # startTime = MyTime.time() - self.getFindTime() + self.seekToTime(log, startTime) + except Exception, e: # pragma: no cover + logSys.error("Error during seek to start time in \"%s\"", filename) + logSys.exception(e) + return False # yoh: has_content is just a bool, so do not expect it to # change -- loop is exited upon break, and is not entered at @@ -717,7 +722,7 @@ class FileFilter(Filter): cntr = 0 unixTime = None lasti = 0 - movecntr = 3 + movecntr = 1 while maxp > minp: i = int(minp + (maxp - minp) / 2) pos = container.seek(i) @@ -726,7 +731,8 @@ class FileFilter(Filter): lncntr = 5; dateTimeMatch = None llen = 0 - i = pos + if lastpos == pos: + i = pos while True: line = container.readline() if not line: @@ -763,6 +769,7 @@ class FileFilter(Filter): lastpos = container.seek(lastFew, False) else: lastpos = container.seek(lastpos, False) + container.setPos(lastpos) if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, (datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr) @@ -952,10 +959,6 @@ class DNSUtils: logSys.warning("Unable to find a corresponding IP address for %s: %s" % (dns, e)) return list() - except socket.error, e: - logSys.warning("Socket error raised trying to resolve hostname %s: %s" - % (dns, e)) - return list() @staticmethod def ipToName(ip): @@ -984,7 +987,7 @@ class DNSUtils: try: socket.inet_aton(s[0]) return True - except socket.error: + except socket.error: # pragma: no cover return False @staticmethod diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index c7dba5ce..7a35bdd9 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -27,7 +27,7 @@ import unittest import getpass import os import sys -import time +import time, datetime import tempfile import uuid @@ -38,7 +38,7 @@ except ImportError: from ..server.jail import Jail from ..server.filterpoll import FilterPoll -from ..server.filter import Filter, FileFilter, DNSUtils +from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils from ..server.failmanager import FailManagerEmpty from ..server.mytime import MyTime from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase @@ -346,6 +346,60 @@ class LogFileFilterPoll(unittest.TestCase): self.assertTrue(self.filter.isModified(LogFileFilterPoll.FILENAME)) self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME)) + def testSeekToTime(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') + tm = lambda time: datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") + time = 1417512352 + f = open(fname, 'w') + fc = FileContainer(fname, self.filter.getLogEncoding()) + fc.open() + fc.setPos(0); self.filter.seekToTime(fc, time) + try: + f.flush() + # empty : + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 0) + # one entry with exact time: + f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + # one entry with smaller time: + f.seek(0) + f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time - 10)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 0) + f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time - 9)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 0) + # add exact time between: + f.write("%s [sshd] error: PAM: failure\n" % tm(time - 1)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 110) + # stil one exact line: + f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time)) + f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 110) + # add something hereafter: + f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 2)) + f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time + 3)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 110) + # add something hereafter: + f.write("%s [sshd] error: PAM: failure\n" % tm(time + 9)) + f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 9)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 110) + + finally: + fc.close() + _killfile(f, fname) class LogFileMonitor(LogCaptureTestCase): """Few more tests for FilterPoll API From 59bf5013c0f7ddcfa88ee638031ec6f79c97393e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Jul 2015 14:58:00 +0200 Subject: [PATCH 128/143] - performance of fail2ban optimized -- cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name for wrong dns or lazy dns-system; -- instead of simple "sleep" used conditional wait "wait_for", that internal increases sleep interval up to sleeptime; -- ticket / banmanager / failmanager modules are performance optimized; -- api of filter (log files), jail, etc. rewritten and extended for performance purposes; - performance of test cases optimized: -- added option "--fast" to decrease wait intervals, avoid passive waiting, and skip few very slow test cases; - code review after partially cherry pick of branch 'ban-time-incr' (see gh-716) -- ticket module prepared to easy merge with newest version of 'ban-time-incr', now additionally holds banTime, banCount and json-data; -- executeCmd partially moved from action to new module utils, etc. -- python 2.6 compatibility; - testExecuteTimeoutWithNastyChildren: test case repaired - wait for pid file inside bash, kill tree in any case (gh-1155); - testSocket: test case repaired - wait for server thread starts a socket (listener) --- MANIFEST | 2 +- bin/fail2ban-testcases | 5 +- doc/fail2ban.server.rst | 2 +- ...faildata.rst => fail2ban.server.utils.rst} | 4 +- fail2ban/server/action.py | 73 +----- fail2ban/server/actions.py | 12 +- fail2ban/server/banmanager.py | 37 ++- fail2ban/server/database.py | 13 +- fail2ban/server/faildata.py | 71 ----- fail2ban/server/failmanager.py | 38 +-- fail2ban/server/filter.py | 70 +++-- fail2ban/server/filtergamin.py | 26 +- fail2ban/server/filterpoll.py | 68 ++--- fail2ban/server/jailthread.py | 7 +- fail2ban/server/server.py | 12 +- fail2ban/server/ticket.py | 145 +++++++++-- fail2ban/server/transmitter.py | 2 +- fail2ban/server/utils.py | 242 ++++++++++++++++++ fail2ban/tests/action_d/test_badips.py | 2 + fail2ban/tests/action_d/test_smtp.py | 2 + fail2ban/tests/actionstestcase.py | 10 +- fail2ban/tests/actiontestcase.py | 59 +++-- fail2ban/tests/banmanagertestcase.py | 1 + fail2ban/tests/databasetestcase.py | 13 +- fail2ban/tests/dummyjail.py | 23 +- fail2ban/tests/filtertestcase.py | 70 ++--- fail2ban/tests/servertestcase.py | 47 ++-- fail2ban/tests/sockettestcase.py | 17 +- fail2ban/tests/utils.py | 85 +++--- 29 files changed, 768 insertions(+), 390 deletions(-) rename doc/{fail2ban.server.faildata.rst => fail2ban.server.utils.rst} (55%) delete mode 100644 fail2ban/server/faildata.py create mode 100644 fail2ban/server/utils.py diff --git a/MANIFEST b/MANIFEST index d6bdba10..7306cc41 100644 --- a/MANIFEST +++ b/MANIFEST @@ -180,7 +180,6 @@ fail2ban/server/banmanager.py fail2ban/server/database.py fail2ban/server/datedetector.py fail2ban/server/datetemplate.py -fail2ban/server/faildata.py fail2ban/server/failmanager.py fail2ban/server/failregex.py fail2ban/server/filter.py @@ -197,6 +196,7 @@ fail2ban/server/server.py fail2ban/server/strptime.py fail2ban/server/ticket.py fail2ban/server/transmitter.py +fail2ban/server/utils.py fail2ban/tests/__init__.py fail2ban/tests/action_d/__init__.py fail2ban/tests/action_d/test_badips.py diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index dd6547a5..28af8265 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -58,6 +58,9 @@ def get_opt_parser(): Option('-n', "--no-network", action="store_true", dest="no_network", help="Do not run tests that require the network"), + Option('-f', "--fast", action="store_true", + dest="fast", + help="Try to increase speed of the tests, decreasing of wait intervals, memory database"), Option("-t", "--log-traceback", action='store_true', help="Enrich log-messages with compressed tracebacks"), Option("--full-traceback", action='store_true', @@ -120,7 +123,7 @@ if not opts.log_level or opts.log_level != 'critical': # pragma: no cover print("Fail2ban %s test suite. Python %s. Please wait..." \ % (version, str(sys.version).replace('\n', ''))) -tests = gatherTests(regexps, opts.no_network) +tests = gatherTests(regexps, opts) # # Run the tests # diff --git a/doc/fail2ban.server.rst b/doc/fail2ban.server.rst index fdc3bd20..dc5d1dee 100644 --- a/doc/fail2ban.server.rst +++ b/doc/fail2ban.server.rst @@ -10,7 +10,6 @@ fail2ban.server package fail2ban.server.database fail2ban.server.datedetector fail2ban.server.datetemplate - fail2ban.server.faildata fail2ban.server.failmanager fail2ban.server.failregex fail2ban.server.filter @@ -26,3 +25,4 @@ fail2ban.server package fail2ban.server.strptime fail2ban.server.ticket fail2ban.server.transmitter + fail2ban.server.utils diff --git a/doc/fail2ban.server.faildata.rst b/doc/fail2ban.server.utils.rst similarity index 55% rename from doc/fail2ban.server.faildata.rst rename to doc/fail2ban.server.utils.rst index c11d8208..3061e632 100644 --- a/doc/fail2ban.server.faildata.rst +++ b/doc/fail2ban.server.utils.rst @@ -1,7 +1,7 @@ -fail2ban.server.faildata module +fail2ban.server.utils module =============================== -.. automodule:: fail2ban.server.faildata +.. automodule:: fail2ban.server.utils :members: :undoc-members: :show-inheritance: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 85bdd166..446ae853 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -32,6 +32,7 @@ import time from abc import ABCMeta from collections import MutableMapping +from .utils import Utils from ..helpers import getLogger # Gets the instance of the logger. @@ -40,21 +41,6 @@ logSys = getLogger(__name__) # Create a lock for running system commands _cmd_lock = threading.Lock() -# Some hints on common abnormal exit codes -_RETCODE_HINTS = { - 127: '"Command not found". Make sure that all commands in %(realCmd)r ' - 'are in the PATH of fail2ban-server process ' - '(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). ' - 'You may want to start ' - '"fail2ban-server -f" separately, initiate it with ' - '"fail2ban-client reload" in another shell session and observe if ' - 'additional informative error messages appear in the terminals.' - } - -# Dictionary to lookup signal name from number -signame = dict((num, name) - for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) - class CallingMap(MutableMapping): """A Mapping type which returns the result of callable values. @@ -561,61 +547,6 @@ class CommandAction(ActionBase): _cmd_lock.acquire() try: - retcode = None # to guarantee being defined upon early except - stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") - stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") - - popen = subprocess.Popen( - realCmd, stdout=stdout, stderr=stderr, shell=True, - preexec_fn=os.setsid # so that killpg does not kill our process - ) - stime = time.time() - retcode = popen.poll() - while time.time() - stime <= timeout and retcode is None: - time.sleep(0.1) - retcode = popen.poll() - if retcode is None: - logSys.error("%s -- timed out after %i seconds." % - (realCmd, timeout)) - pgid = os.getpgid(popen.pid) - os.killpg(pgid, signal.SIGTERM) # Terminate the process - time.sleep(0.1) - retcode = popen.poll() - if retcode is None: # Still going... - os.killpg(pgid, signal.SIGKILL) # Kill the process - time.sleep(0.1) - retcode = popen.poll() - except OSError as e: - logSys.error("%s -- failed with %s" % (realCmd, e)) + return Utils.executeCmd(realCmd, timeout, shell=True, output=False) finally: _cmd_lock.release() - - std_level = retcode == 0 and logging.DEBUG or logging.ERROR - if std_level >= logSys.getEffectiveLevel(): - stdout.seek(0); msg = stdout.read() - if msg != '': - logSys.log(std_level, "%s -- stdout: %r", realCmd, msg) - stderr.seek(0); msg = stderr.read() - if msg != '': - logSys.log(std_level, "%s -- stderr: %r", realCmd, msg) - stdout.close() - stderr.close() - - if retcode == 0: - logSys.debug("%s -- returned successfully" % realCmd) - return True - elif retcode is None: - logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid)) - elif retcode < 0 or retcode > 128: - # dash would return negative while bash 128 + n - sigcode = -retcode if retcode < 0 else retcode - 128 - logSys.error("%s -- killed with %s (return code: %s)" % - (realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)) - else: - msg = _RETCODE_HINTS.get(retcode, None) - logSys.error("%s -- returned %i" % (realCmd, retcode)) - if msg: - logSys.info("HINT on %i: %s" - % (retcode, msg % locals())) - return False - diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c00dc7fc..5469722f 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -42,6 +42,7 @@ from .banmanager import BanManager from .jailthread import JailThread from .action import ActionBase, CommandAction, CallingMap from .mytime import MyTime +from .utils import Utils from ..helpers import getLogger # Gets the instance of the logger. @@ -225,14 +226,11 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if not self.idle: - #logSys.debug(self._jail.name + ": action") - ret = self.__checkBan() - if not ret: - self.__checkUnBan() - time.sleep(self.sleeptime) - else: + if self.idle: time.sleep(self.sleeptime) + continue + if not Utils.wait_for(self.__checkBan, self.sleeptime): + self.__checkUnBan() self.__flushBan() actions = self._actions.items() diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 662666b0..a958a5c8 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -247,12 +247,10 @@ class BanManager: @staticmethod def createBanTicket(ticket): - ip = ticket.getIP() - #lastTime = ticket.getTime() - lastTime = MyTime.time() - banTicket = BanTicket(ip, lastTime, ticket.getMatches()) - banTicket.setAttempt(ticket.getAttempt()) - return banTicket + # we should always use correct time to calculate correct end time (ban time is variable now, + # + possible double banning by restore from database and from log file) + # so use as lastTime always time from ticket. + return BanTicket(ticket=ticket) ## # Add a ban ticket. @@ -264,11 +262,25 @@ class BanManager: def addBanTicket(self, ticket): try: self.__lock.acquire() - if not self._inBanList(ticket): - self.__banList.append(ticket) - self.__banTotal += 1 - return True - return False + # check already banned + for i in self.__banList: + if ticket.getIP() == i.getIP(): + # if already permanent + btorg, torg = i.getBanTime(self.__banTime), i.getTime() + if btorg == -1: + return False + # if given time is less than already banned time + btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime() + if btnew != -1 and tnew + btnew <= torg + btorg: + return False + # we have longest ban - set new (increment) ban time + i.setTime(tnew) + i.setBanTime(btnew) + return False + # not yet banned - add new + self.__banList.append(ticket) + self.__banTotal += 1 + return True finally: self.__lock.release() @@ -313,8 +325,7 @@ class BanManager: return list() # Gets the list of ticket to remove. - unBanList = [ticket for ticket in self.__banList - if ticket.getTime() < time - self.__banTime] + unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)] # Removes tickets. self.__banList = [ticket for ticket in self.__banList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 3cfaa5a6..c6e8c95c 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -418,8 +418,7 @@ class Fail2BanDb(object): cur.execute( "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", (jail.name, ticket.getIP(), int(round(ticket.getTime())), - {"matches": ticket.getMatches(), - "failures": ticket.getAttempt()})) + ticket.getData())) @commitandrollback def delBan(self, cur, jail, ip): @@ -477,8 +476,8 @@ class Fail2BanDb(object): tickets = [] for ip, timeofban, data in self._getBans(**kwargs): #TODO: Implement data parts once arbitrary match keys completed - tickets.append(FailTicket(ip, timeofban, data.get('matches'))) - tickets[-1].setAttempt(data.get('failures', 1)) + tickets.append(FailTicket(ip, timeofban)) + tickets[-1].setData(data) return tickets def getBansMerged(self, ip=None, jail=None, bantime=None): @@ -520,6 +519,7 @@ class Fail2BanDb(object): prev_banip = results[0][0] matches = [] failures = 0 + tickdata = {} for banip, timeofban, data in results: #TODO: Implement data parts once arbitrary match keys completed if banip != prev_banip: @@ -530,11 +530,14 @@ class Fail2BanDb(object): prev_banip = banip matches = [] failures = 0 - matches.extend(data.get('matches', [])) + tickdata = {} + matches.extend(data.get('matches', ())) failures += data.get('failures', 1) + tickdata.update(data.get('data', {})) prev_timeofban = timeofban ticket = FailTicket(banip, prev_timeofban, matches) ticket.setAttempt(failures) + ticket.setData(**tickdata) tickets.append(ticket) if cacheKey: diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py deleted file mode 100644 index 2dd8d4d8..00000000 --- a/fail2ban/server/faildata.py +++ /dev/null @@ -1,71 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -# Author: Cyril Jaquier -# - -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" -__license__ = "GPL" - -from ..helpers import getLogger - -# Gets the instance of the logger. -logSys = getLogger(__name__) - - -class FailData: - - def __init__(self): - self.__retry = 0 - self.__lastTime = 0 - self.__lastReset = 0 - self.__matches = [] - - def setRetry(self, value): - self.__retry = value - # keep only the last matches or reset entirely - # Explicit if/else for compatibility with Python 2.4 - if value: - self.__matches = self.__matches[-min(len(self.__matches, value)):] - else: - self.__matches = [] - - def getRetry(self): - return self.__retry - - def getMatches(self): - return self.__matches - - def inc(self, matches=None): - self.__retry += 1 - self.__matches += matches or [] - - def setLastTime(self, value): - if value > self.__lastTime: - self.__lastTime = value - - def getLastTime(self): - return self.__lastTime - - def getLastReset(self): - return self.__lastReset - - def setLastReset(self, value): - self.__lastReset = value diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 37a5fe55..4d006098 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -27,7 +27,6 @@ __license__ = "GPL" from threading import Lock import logging -from .faildata import FailData from .ticket import FailTicket from ..helpers import getLogger @@ -86,26 +85,35 @@ class FailManager: finally: self.__lock.release() - def addFailure(self, ticket): + def addFailure(self, ticket, count=1): + attempts = 1 try: self.__lock.acquire() ip = ticket.getIP() - unixTime = ticket.getTime() - matches = ticket.getMatches() if ip in self.__failList: fData = self.__failList[ip] + # if the same object: + if fData is ticket: + matches = None + else: + matches = ticket.getMatches() + unixTime = ticket.getTime() if fData.getLastReset() < unixTime - self.__maxTime: fData.setLastReset(unixTime) fData.setRetry(0) - fData.inc(matches) + fData.inc(matches, 1, count) fData.setLastTime(unixTime) else: - fData = FailData() - fData.inc(matches) - fData.setLastReset(unixTime) - fData.setLastTime(unixTime) + # if already FailTicket - add it direct, otherwise create (using copy all ticket data): + if isinstance(ticket, FailTicket): + fData = ticket; + else: + fData = FailTicket(ticket=ticket) + if count > ticket.getAttempt(): + fData.setRetry(count) self.__failList[ip] = fData + attempts = fData.getRetry() self.__failTotal += 1 if logSys.getEffectiveLevel() <= logging.DEBUG: @@ -118,6 +126,7 @@ class FailManager: % (self.__failTotal, len(self.__failList), failures_summary)) finally: self.__lock.release() + return attempts def size(self): try: @@ -140,17 +149,14 @@ class FailManager: if ip in self.__failList: del self.__failList[ip] - def toBan(self): + def toBan(self, ip=None): try: self.__lock.acquire() - for ip in self.__failList: + for ip in ([ip] if ip != None and ip in self.__failList else self.__failList): data = self.__failList[ip] if data.getRetry() >= self.__maxRetry: - self.__delFailure(ip) - # Create a FailTicket from BanData - failTicket = FailTicket(ip, data.getLastTime(), data.getMatches()) - failTicket.setAttempt(data.getRetry()) - return failTicket + del self.__failList[ip] + return data raise FailManagerEmpty finally: self.__lock.release() diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 621f826b..0a2a9949 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" import codecs +import datetime import fcntl import locale import logging @@ -316,13 +317,12 @@ class Filter(JailThread): logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip) unixTime = MyTime.time() - for i in xrange(self.failManager.getMaxRetry()): - self.failManager.addFailure(FailTicket(ip, unixTime)) + self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry()) # Perform the banning of the IP now. try: # pragma: no branch - exception is the only way out while True: - ticket = self.failManager.toBan() + ticket = self.failManager.toBan(ip) self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) @@ -427,17 +427,19 @@ class Filter(JailThread): ip = element[1] unixTime = element[2] lines = element[3] - logSys.debug("Processing line with time:%s and ip:%s" - % (unixTime, ip)) + logSys.debug("Processing line with time:%s and ip:%s", + unixTime, ip) if unixTime < MyTime.time() - self.getFindTime(): - logSys.debug("Ignore line since time %s < %s - %s" - % (unixTime, MyTime.time(), self.getFindTime())) + logSys.debug("Ignore line since time %s < %s - %s", + unixTime, MyTime.time(), self.getFindTime()) break if self.inIgnoreIPList(ip, log_ignore=True): continue - logSys.info("[%s] Found %s" % (self.jail.name, ip)) - ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) - self.failManager.addFailure(FailTicket(ip, unixTime, lines)) + logSys.info( + "[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") + ) + tick = FailTicket(ip, unixTime, lines) + self.failManager.addFailure(tick) ## # Returns true if the line should be ignored. @@ -606,6 +608,14 @@ class FileFilter(Filter): # to be overridden by backends pass + ## + # Get the log file names + # + # @return log paths + + def getLogPaths(self): + return self.__logs.keys() + ## # Get the log containers # @@ -614,6 +624,14 @@ class FileFilter(Filter): def getLogs(self): return self.__logs.values() + ## + # Get the count of log containers + # + # @return count of log containers + + def getLogCount(self): + return len(self.__logs) + ## # Check whether path is already monitored. # @@ -941,32 +959,50 @@ class JournalFilter(Filter): # pragma: systemd no cover import socket import struct +from .utils import Utils class DNSUtils: IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$") + # todo: make configurable the expired time and max count of cache entries: + CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=60*60) + CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=60*60) + @staticmethod def dnsToIp(dns): """ Convert a DNS into an IP address using the Python socket module. Thanks to Kevin Drapel. """ + # cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system: + v = DNSUtils.CACHE_dnsToIp.get(dns) + if v is not None: + return v # retrieve ip (todo: use AF_INET6 for IPv6) try: - return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)]) + v = set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)]) except socket.error, e: - logSys.warning("Unable to find a corresponding IP address for %s: %s" - % (dns, e)) - return list() + # todo: make configurable the expired time of cache entry: + logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e) + v = list() + DNSUtils.CACHE_dnsToIp.set(dns, v) + return v @staticmethod def ipToName(ip): + # cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns: + v = DNSUtils.CACHE_ipToName.get(ip) + if v is not None: + return v + # retrieve name try: - return socket.gethostbyaddr(ip)[0] + v = socket.gethostbyaddr(ip)[0] except socket.error, e: - logSys.debug("Unable to find a name for the IP %s: %s" % (ip, e)) - return None + logSys.debug("Unable to find a name for the IP %s: %s", ip, e) + v = None + DNSUtils.CACHE_ipToName.set(ip, v) + return v @staticmethod def searchIP(text): diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index e731a8e9..b094cfeb 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -31,6 +31,7 @@ import gamin from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime +from .utils import Utils from ..helpers import getLogger # Gets the instance of the logger. @@ -102,6 +103,15 @@ class FilterGamin(FileFilter): def _delLogPath(self, path): self.monitor.stop_watch(path) + def _handleEvents(self): + ret = False + mon = self.monitor + while mon and mon.event_pending(): + mon.handle_events() + mon = self.monitor + ret = True + return ret + ## # Main loop. # @@ -112,12 +122,10 @@ class FilterGamin(FileFilter): def run(self): # Gamin needs a loop to collect and dispatch events while self.active: - if not self.idle: - # We cannot block here because we want to be able to - # exit. - if self.monitor.event_pending(): - self.monitor.handle_events() - time.sleep(self.sleeptime) + if self.idle: + time.sleep(self.sleeptime) + continue + Utils.wait_for(self._handleEvents, self.sleeptime) logSys.debug(self.jail.name + ": filter terminated") return True @@ -129,6 +137,6 @@ class FilterGamin(FileFilter): # Desallocates the resources used by Gamin. def __cleanup(self): - for log in self.getLogs(): - self.monitor.stop_watch(log.getFileName()) - del self.monitor + for filename in self.getLogPaths(): + self.monitor.stop_watch(filename) + self.monitor = None diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 0e99455e..1afda7f2 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -31,6 +31,7 @@ from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime from ..helpers import getLogger +from ..server.utils import Utils # Gets the instance of the logger. logSys = getLogger(__name__) @@ -78,6 +79,15 @@ class FilterPoll(FileFilter): del self.__prevStats[path] del self.__file404Cnt[path] + ## + # Get a modified log path at once + # + def getModified(self, modlst): + for filename in self.getLogPaths(): + if self.isModified(filename): + modlst.append(filename) + return modlst + ## # Main loop. # @@ -89,31 +99,32 @@ class FilterPoll(FileFilter): while self.active: if logSys.getEffectiveLevel() <= 6: logSys.log(6, "Woke up idle=%s with %d files monitored", - self.idle, len(self.getLogs())) - if not self.idle: - # Get file modification - for container in self.getLogs(): - filename = container.getFileName() - if self.isModified(filename): - # set start time as now - find time for first usage only (prevent performance bug with polling of big files) - self.getFailures(filename, - (MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None - ) - self.__initial[filename] = True - self.__modified = True + self.idle, self.getLogCount()) + if self.idle: + if not Utils.wait_for(lambda: not self.idle, + self.sleeptime * 100, self.sleeptime + ): + continue + # Get file modification + modlst = [] + Utils.wait_for(lambda: self.getModified(modlst), self.sleeptime) + for filename in modlst: + # set start time as now - find time for first usage only (prevent performance bug with polling of big files) + self.getFailures(filename, + (MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None + ) + self.__initial[filename] = True + self.__modified = True - if self.__modified: - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.dateDetector.sortTemplate() - self.__modified = False - time.sleep(self.sleeptime) - else: - time.sleep(self.sleeptime) + if self.__modified: + try: + while True: + ticket = self.failManager.toBan() + self.jail.putFailTicket(ticket) + except FailManagerEmpty: + self.failManager.cleanup(MyTime.time()) + self.dateDetector.sortTemplate() + self.__modified = False logSys.debug( (self.jail is not None and self.jail.name or "jailless") + " filter terminated") @@ -129,7 +140,7 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) stats = logStats.st_mtime, logStats.st_ino, logStats.st_size - pstats = self.__prevStats[filename] + pstats = self.__prevStats.get(filename, ()) self.__file404Cnt[filename] = 0 if logSys.getEffectiveLevel() <= 7: # we do not want to waste time on strftime etc if not necessary @@ -139,10 +150,9 @@ class FilterPoll(FileFilter): # os.system("stat %s | grep Modify" % filename) if pstats == stats: return False - else: - logSys.debug("%s has been modified", filename) - self.__prevStats[filename] = stats - return True + logSys.debug("%s has been modified", filename) + self.__prevStats[filename] = stats + return True except OSError, e: logSys.error("Unable to get stat on %s because of: %s" % (filename, e)) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 3897801a..eb43e453 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -28,6 +28,7 @@ import sys from threading import Thread from abc import abstractmethod +from .utils import Utils from ..helpers import excepthook @@ -48,14 +49,14 @@ class JailThread(Thread): The time the thread sleeps for in the loop. """ - def __init__(self): - super(JailThread, self).__init__() + def __init__(self, name=None): + super(JailThread, self).__init__(name=name) ## Control the state of the thread. self.active = False ## Control the idle state of the thread. self.idle = False ## The time the thread sleeps in the loop. - self.sleeptime = 1 + self.sleeptime = Utils.DEFAULT_SLEEP_TIME # excepthook workaround for threads, derived from: # http://bugs.python.org/issue1230540#msg91244 diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 9836b61d..1b335d02 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -211,8 +211,7 @@ class Server: def getLogPath(self, name): filter_ = self.__jails[name].filter if isinstance(filter_, FileFilter): - return [m.getFileName() - for m in filter_.getLogs()] + return filter_.getLogPaths() else: # pragma: systemd no cover logSys.info("Jail %s is not a FileFilter instance" % name) return [] @@ -324,6 +323,15 @@ class Server: def getBanTime(self, name): return self.__jails[name].actions.getBanTime() + def is_alive(self, jailnum=None): + if jailnum is not None and len(self.__jails) != jailnum: + return 0 + for j in self.__jails: + j = self.__jails[j] + if not j.is_alive(): + return 0 + return 1 + # Status def status(self): try: diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 70be06fe..49ebf9ea 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -24,7 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import sys + from ..helpers import getLogger +from .mytime import MyTime # Gets the instance of the logger. logSys = getLogger(__name__) @@ -32,7 +35,7 @@ logSys = getLogger(__name__) class Ticket: - def __init__(self, ip, time, matches=None): + def __init__(self, ip=None, time=None, matches=None, ticket=None): """Ticket constructor @param ip the IP address @@ -41,14 +44,21 @@ class Ticket: """ self.setIP(ip) - self.__time = time - self.__attempt = 0 - self.__file = None - self.__matches = matches or [] + self._flags = 0; + self._banCount = 0; + self._banTime = None; + self._time = time if time is not None else MyTime.time() + self._data = {'matches': [], 'failures': 0} + if ticket: + # ticket available - copy whole information from ticket: + self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__) + else: + self._data['matches'] = matches or [] def __str__(self): return "%s: ip=%s time=%s #attempts=%d matches=%r" % \ - (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches) + (self.__class__.__name__.split('.')[-1], self.__ip, self._time, + self._data['failures'], self._data.get('matches', [])) def __repr__(self): return str(self) @@ -56,9 +66,8 @@ class Ticket: def __eq__(self, other): try: return self.__ip == other.__ip and \ - round(self.__time, 2) == round(other.__time, 2) and \ - self.__attempt == other.__attempt and \ - self.__matches == other.__matches + round(self._time, 2) == round(other._time, 2) and \ + self._data == other._data except AttributeError: return False @@ -72,24 +81,128 @@ class Ticket: return self.__ip def setTime(self, value): - self.__time = value + self._time = value def getTime(self): - return self.__time - + return self._time + + def setBanTime(self, value): + self._banTime = value; + + def getBanTime(self, defaultBT = None): + return (self._banTime if not self._banTime is None else defaultBT); + + def setBanCount(self, value): + self._banCount = value; + + def incrBanCount(self, value = 1): + self._banCount += value; + + def getBanCount(self): + return self._banCount; + + def isTimedOut(self, time, defaultBT = None): + bantime = (self._banTime if not self._banTime is None else defaultBT); + # permanent + if bantime == -1: + return False + # timed out + return (time > self._time + bantime) + def setAttempt(self, value): - self.__attempt = value + self._data['failures'] = value def getAttempt(self): - return self.__attempt + return self._data['failures'] + + def setMatches(self, matches): + self._data['matches'] = matches or [] def getMatches(self): - return self.__matches + return self._data.get('matches', []) + + def setData(self, *args, **argv): + # if overwrite - set data and filter None values: + if len(args) == 1: + # todo: if support >= 2.7 only: + # self._data = {k:v for k,v in args[0].iteritems() if v is not None} + self._data = dict([(k,v) for k,v in args[0].iteritems() if v is not None]) + # add k,v list or dict (merge): + elif len(args) == 2: + self._data.update((args,)) + elif len(args) > 2: + self._data.update((k,v) for k,v in zip(*[iter(args)]*2)) + if len(argv): + self._data.update(argv) + # filter (delete) None values: + # todo: if support >= 2.7 only: + # self._data = {k:v for k,v in self._data.iteritems() if v is not None} + self._data = dict([(k,v) for k,v in self._data.iteritems() if v is not None]) + + def getData(self, key=None, default=None): + # return whole data dict: + if key is None: + return self._data + # return default if not exists: + if not self._data: + return default + # return filtered by lambda/function: + if callable(key): + # todo: if support >= 2.7 only: + # return {k:v for k,v in self._data.iteritems() if key(k)} + return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) + # return filtered by keys: + if hasattr(key, '__iter__'): + # todo: if support >= 2.7 only: + # return {k:v for k,v in self._data.iteritems() if k in key} + return dict([(k,v) for k,v in self._data.iteritems() if k in key]) + # return single value of data: + return self._data.get(key, default) class FailTicket(Ticket): - pass + def __init__(self, ip=None, time=None, matches=None, ticket=None): + # this class variables: + self.__retry = 0 + self.__lastReset = None + # create/copy using default ticket constructor: + Ticket.__init__(self, ip, time, matches, ticket) + # init: + if ticket is None: + self.__lastReset = time if time is not None else self.getTime() + if not self.__retry: + self.__retry = self._data['failures']; + + def setRetry(self, value): + self.__retry = value + if not self._data['failures']: + self._data['failures'] = 1 + if not value: + self._data['failures'] = 0 + self._data['matches'] = [] + + def getRetry(self): + return max(self.__retry, self._data['failures']) + + def inc(self, matches=None, attempt=1, count=1): + self.__retry += count + self._data['failures'] += attempt + if matches: + self._data['matches'] += matches + + def setLastTime(self, value): + if value > self._time: + self._time = value + + def getLastTime(self): + return self._time + + def getLastReset(self): + return self.__lastReset + + def setLastReset(self, value): + self.__lastReset = value ## # Ban Ticket. diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index eb5b7713..4c4c32f7 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -95,7 +95,7 @@ class Transmitter: return None elif command[0] == "sleep": value = command[1] - time.sleep(int(value)) + time.sleep(float(value)) return None elif command[0] == "flushlogs": return self.__server.flushLogs() diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py new file mode 100644 index 00000000..7e69ddca --- /dev/null +++ b/fail2ban/server/utils.py @@ -0,0 +1,242 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +__author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester" +__license__ = "GPL" + +import logging, os, fcntl, subprocess, time, signal +from ..helpers import getLogger + +# Gets the instance of the logger. +logSys = getLogger(__name__) + +# Some hints on common abnormal exit codes +_RETCODE_HINTS = { + 127: '"Command not found". Make sure that all commands in %(realCmd)r ' + 'are in the PATH of fail2ban-server process ' + '(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). ' + 'You may want to start ' + '"fail2ban-server -f" separately, initiate it with ' + '"fail2ban-client reload" in another shell session and observe if ' + 'additional informative error messages appear in the terminals.' + } + +# Dictionary to lookup signal name from number +signame = dict((num, name) + for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) + +class Utils(): + """Utilities provide diverse static methods like executes OS shell commands, etc. + """ + + DEFAULT_SLEEP_TIME = 0.1 + DEFAULT_SLEEP_INTERVAL = 0.01 + + + class Cache(dict): + + def __init__(self, maxCount=1000, maxTime=60*60): + self.maxCount = maxCount + self.maxTime = maxTime + + def get(self, k, defv=None): + v = dict.get(self, k) + if v: + if v[1] > time.time(): + return v[0] + del self[k] + return defv + + def set(self, k, v): + t = time.time() + # clean cache if max count reached: + if len(self) >= self.maxCount: + for (ck,cv) in self.items(): + if cv[1] < t: + del self[ck] + # if still max count - remove any one: + if len(self) >= self.maxCount: + self.popitem() + self[k] = (v, t + self.maxTime) + + + @staticmethod + def setFBlockMode(fhandle, value): + flags = fcntl.fcntl(fhandle, fcntl.F_GETFL) + if not value: + flags |= os.O_NONBLOCK + else: + flags &= ~os.O_NONBLOCK + fcntl.fcntl(fhandle, fcntl.F_SETFL, flags) + return flags + + @staticmethod + def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True): + """Executes a command. + + Parameters + ---------- + realCmd : str + The command to execute. + timeout : int + The time out in seconds for the command. + shell : bool + If shell is True (default), the specified command (may be a string) will be + executed through the shell. + output : bool + If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode) + + Returns + ------- + bool + True if the command succeeded. + + Raises + ------ + OSError + If command fails to be executed. + RuntimeError + If command execution times out. + """ + stdout = stderr = None + retcode = None + if not callable(timeout): + stime = time.time() + timeout_expr = lambda: time.time() - stime <= timeout + else: + timeout_expr = timeout + try: + popen = subprocess.Popen( + realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, + preexec_fn=os.setsid # so that killpg does not kill our process + ) + retcode = popen.poll() + while retcode is None and timeout_expr(): + time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) + retcode = popen.poll() + if retcode is None: + logSys.error("%s -- timed out after %s seconds." % + (realCmd, timeout)) + pgid = os.getpgid(popen.pid) + # if not tree - first try to terminate and then kill, otherwise - kill (-9) only: + os.killpg(pgid, signal.SIGTERM) # Terminate the process + time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) + retcode = popen.poll() + #logSys.debug("%s -- terminated %s ", realCmd, retcode) + if retcode is None or tout_kill_tree: # Still going... + os.killpg(pgid, signal.SIGKILL) # Kill the process + time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) + retcode = popen.poll() + #logSys.debug("%s -- killed %s ", realCmd, retcode) + if retcode is None and not Utils.pid_exists(pgid): + retcode = signal.SIGKILL + except OSError as e: + logSys.error("%s -- failed with %s" % (realCmd, e)) + + std_level = retcode == 0 and logging.DEBUG or logging.ERROR + # if we need output (to return or to log it): + if output or std_level >= logSys.getEffectiveLevel(): + # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode. + if popen.stdout: + try: + if retcode < 0: + Utils.setFBlockMode(popen.stdout, False) + stdout = popen.stdout.read() + except IOError as e: + logSys.error(" ... -- failed to read stdout %s", e) + if stdout is not None and stdout != '': + logSys.log(std_level, "%s -- stdout: %r", realCmd, stdout) + popen.stdout.close() + if popen.stderr: + try: + if retcode < 0: + Utils.setFBlockMode(popen.stderr, False) + stderr = popen.stderr.read() + except IOError as e: + logSys.error(" ... -- failed to read stderr %s", e) + if stderr is not None and stderr != '': + logSys.log(std_level, "%s -- stderr: %r", realCmd, stderr) + popen.stderr.close() + + if retcode == 0: + logSys.debug("%s -- returned successfully", realCmd) + return True if not output else (True, stdout, stderr, retcode) + elif retcode is None: + logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid)) + elif retcode < 0 or retcode > 128: + # dash would return negative while bash 128 + n + sigcode = -retcode if retcode < 0 else retcode - 128 + logSys.error("%s -- killed with %s (return code: %s)" % + (realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)) + else: + msg = _RETCODE_HINTS.get(retcode, None) + logSys.error("%s -- returned %i" % (realCmd, retcode)) + if msg: + logSys.info("HINT on %i: %s", retcode, msg % locals()) + return False if not output else (False, stdout, stderr, retcode) + + @staticmethod + def wait_for(cond, timeout, interval=None): + """Wait until condition expression `cond` is True, up to `timeout` sec + """ + ini = 1 + while True: + ret = cond() + if ret: + return ret + if ini: + ini = stm = 0 + time0 = time.time() + timeout + if not interval: + interval = Utils.DEFAULT_SLEEP_INTERVAL + if time.time() > time0: + break + stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME) + time.sleep(stm) + return ret + + # Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid + # under cc by-sa 3.0 + if os.name == 'posix': + @staticmethod + def pid_exists(pid): + """Check whether pid exists in the current process table.""" + import errno + if pid < 0: + return False + try: + os.kill(pid, 0) + except OSError as e: + return e.errno == errno.EPERM + else: + return True + else: + @staticmethod + def pid_exists(pid): + import ctypes + kernel32 = ctypes.windll.kernel32 + SYNCHRONIZE = 0x100000 + + process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid) + if process != 0: + kernel32.CloseHandle(process) + return True + else: + return False diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index a7f148b1..74594420 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -29,6 +29,8 @@ if sys.version_info >= (2,7): def setUp(self): """Call before every test case.""" + unittest.F2B.SkipIfNoNetwork() + self.jail = DummyJail() self.jail.actions.add("test") diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 35ac2393..27442832 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -46,6 +46,8 @@ class SMTPActionTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + unittest.F2B.SkipIfNoNetwork() + self.jail = DummyJail() pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py") pythonModuleName = os.path.basename(pythonModule.rstrip(".py")) diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index 0ceb35d5..dff0d4c6 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -30,6 +30,7 @@ import tempfile from ..server.actions import Actions from ..server.ticket import FailTicket +from ..server.utils import Utils from .dummyjail import DummyJail from .utils import LogCaptureTestCase @@ -81,8 +82,7 @@ class ExecuteActions(LogCaptureTestCase): self.defaultActions() self.__actions.start() with open(self.__tmpfilename) as f: - time.sleep(3) - self.assertEqual(f.read(),"ip start 64\n") + self.assertTrue( Utils.wait_for(lambda: (f.read() == "ip start 64\n"), 3) ) self.__actions.stop() self.__actions.join() @@ -97,8 +97,7 @@ class ExecuteActions(LogCaptureTestCase): self.assertLogged("TestAction initialised") self.__actions.start() - time.sleep(3) - self.assertLogged("TestAction action start") + self.assertTrue( Utils.wait_for(lambda: self._is_logged("TestAction action start"), 3) ) self.__actions.stop() self.__actions.join() @@ -135,8 +134,7 @@ class ExecuteActions(LogCaptureTestCase): "action.d/action_errors.py"), {}) self.__actions.start() - time.sleep(3) - self.assertLogged("Failed to start") + self.assertTrue( Utils.wait_for(lambda: self._is_logged("Failed to start"), 3) ) self.__actions.stop() self.__actions.join() self.assertLogged("Failed to stop") diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 289d8896..8c9b5ef1 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -25,10 +25,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import os -import time import tempfile +import time +import unittest from ..server.action import CommandAction, CallingMap +from ..server.utils import Utils from .utils import LogCaptureTestCase from .utils import pid_exists @@ -194,16 +196,17 @@ class CommandActionTest(LogCaptureTestCase): self.assertLogged('HINT on 127: "Command not found"') def testExecuteTimeout(self): + unittest.F2B.SkipIfFast() stime = time.time() # Should take a minute - self.assertFalse(CommandAction.executeCmd('sleep 60', timeout=2)) + self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=1)) # give a test still 1 second, because system could be too busy - self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) + self.assertTrue(time.time() >= stime + 1 and time.time() <= stime + 2) self.assertLogged( - 'sleep 60 -- timed out after 2 seconds', - 'sleep 60 -- timed out after 3 seconds' + 'sleep 30 -- timed out after 1 seconds', + 'sleep 30 -- timed out after 2 seconds' ) - self.assertLogged('sleep 60 -- killed with SIGTERM') + self.assertLogged('sleep 30 -- killed with SIGTERM') def testExecuteTimeoutWithNastyChildren(self): # temporary file for a nasty kid shell script @@ -215,29 +218,53 @@ class CommandActionTest(LogCaptureTestCase): echo "$$" > %s.pid echo "my pid $$ . sleeping lo-o-o-ong" - sleep 10000 + sleep 30 """ % tmpFilename) + stime = 0 + + # timeout as long as pid-file was not created, but max 5 seconds + def getnasty_tout(): + return ( + getnastypid() is None + and time.time() - stime <= 5 + ) def getnastypid(): - with open(tmpFilename + '.pid') as f: - return int(f.read()) + cpid = None + if os.path.isfile(tmpFilename + '.pid'): + with open(tmpFilename + '.pid') as f: + try: + cpid = int(f.read()) + except ValueError: + pass + return cpid # First test if can kill the bastard + stime = time.time() self.assertFalse(CommandAction.executeCmd( - 'bash %s' % tmpFilename, timeout=.1)) + 'bash %s' % tmpFilename, timeout=getnasty_tout)) + # Wait up to 3 seconds, the child got killed + cpid = getnastypid() # Verify that the process itself got killed - self.assertFalse(pid_exists(getnastypid())) # process should have been killed + self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3)) # process should have been killed + self.assertLogged('my pid ') self.assertLogged('timed out') - self.assertLogged('killed with SIGTERM') + self.assertLogged('killed with SIGTERM', + 'killed with SIGKILL') + os.unlink(tmpFilename + '.pid') # A bit evolved case even though, previous test already tests killing children processes + stime = time.time() self.assertFalse(CommandAction.executeCmd( - 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2)) + 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=getnasty_tout)) + # Wait up to 3 seconds, the child got killed + cpid = getnastypid() # Verify that the process itself got killed - self.assertFalse(pid_exists(getnastypid())) + self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3)) + self.assertLogged('my pid ') self.assertLogged('timed out') - self.assertLogged('killed with SIGTERM') - + self.assertLogged('killed with SIGTERM', + 'killed with SIGKILL') os.unlink(tmpFilename) os.unlink(tmpFilename + '.pid') diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 09d2411e..a2d399b3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -60,6 +60,7 @@ class AddFailure(unittest.TestCase): class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): """Call before every test case.""" + unittest.F2B.SkipIfNoNetwork() self.__ban_ip = "93.184.216.34" self.__asn = "15133" self.__country = "EU" diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 083be2b2..20baa847 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -35,7 +35,12 @@ from ..server.ticket import FailTicket from ..server.actions import Actions from .dummyjail import DummyJail try: - from ..server.database import Fail2BanDb + from ..server.database import Fail2BanDb as Fail2BanDb + # because of tests performance use memory instead of file: + def TestFail2BanDb(filename): + if unittest.F2B.fast: + return Fail2BanDb(':memory:') + return Fail2BanDb(filename) except ImportError: Fail2BanDb = None from .utils import LogCaptureTestCase @@ -55,7 +60,7 @@ class DatabaseTest(LogCaptureTestCase): elif Fail2BanDb is None: return _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") - self.db = Fail2BanDb(self.dbFilename) + self.db = TestFail2BanDb(self.dbFilename) def tearDown(self): """Call after every test case.""" @@ -66,7 +71,7 @@ class DatabaseTest(LogCaptureTestCase): os.remove(self.dbFilename) def testGetFilename(self): - if Fail2BanDb is None: # pragma: no cover + if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover return self.assertEqual(self.dbFilename, self.db.filename) @@ -88,7 +93,7 @@ class DatabaseTest(LogCaptureTestCase): "/this/path/should/not/exist") def testCreateAndReconnect(self): - if Fail2BanDb is None: # pragma: no cover + if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover return self.testAddJail() # Reconnect... diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 6be571d3..bade2f12 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -39,28 +39,27 @@ class DummyJail(Jail, object): self.__actions = Actions(self) def __len__(self): - try: - self.lock.acquire() + with self.lock: return len(self.queue) - finally: - self.lock.release() + + def isEmpty(self): + with self.lock: + return not self.queue + + def isFilled(self): + with self.lock: + return bool(self.queue) def putFailTicket(self, ticket): - try: - self.lock.acquire() + with self.lock: self.queue.append(ticket) - finally: - self.lock.release() def getFailTicket(self): - try: - self.lock.acquire() + with self.lock: try: return self.queue.pop() except IndexError: return False - finally: - self.lock.release() @property def name(self): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 7a35bdd9..59512530 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -41,6 +41,7 @@ from ..server.filterpoll import FilterPoll from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils from ..server.failmanager import FailManagerEmpty from ..server.mytime import MyTime +from ..server.utils import Utils from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase from .dummyjail import DummyJail @@ -162,7 +163,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Opened earlier, therefore must close it fin.close() # to give other threads possibly some time to crunch - time.sleep(0.1) + time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) return fout @@ -299,6 +300,11 @@ class IgnoreIP(LogCaptureTestCase): class IgnoreIPDNS(IgnoreIP): + def setUp(self): + """Call before every test case.""" + unittest.F2B.SkipIfNoNetwork() + IgnoreIP.setUp(self) + def testIgnoreIPDNSOK(self): self.filter.addIgnoreIP("www.epfl.ch") self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12")) @@ -425,16 +431,11 @@ class LogFileMonitor(LogCaptureTestCase): def isModified(self, delay=2.): """Wait up to `delay` sec to assure that it was modified or not """ - time0 = time.time() - while time.time() < time0 + delay: - if self.filter.isModified(self.name): - return True - time.sleep(0.1) - return False + return Utils.wait_for(lambda: self.filter.isModified(self.name), delay) def notModified(self): # shorter wait time for not modified status - return not self.isModified(0.4) + return not self.isModified(4*Utils.DEFAULT_SLEEP_TIME) def testUnaccessibleLogFile(self): os.chmod(self.name, 0) @@ -583,26 +584,21 @@ def get_monitor_failures_testcase(Filter_): #time.sleep(0.2) # Give FS time to ack the removal pass - def isFilled(self, delay=2.): + def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - time0 = time.time() - while time.time() < time0 + delay: - if len(self.jail): - return True - time.sleep(0.1) - return False + return Utils.wait_for(lambda: self.jail.isFilled(), delay) def _sleep_4_poll(self): # Since FilterPoll relies on time stamps and some # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - mtimesleep() + Utils.wait_for(lambda: self.filter.is_alive(), 4*Utils.DEFAULT_SLEEP_TIME) - def isEmpty(self, delay=0.4): + def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): # shorter wait time for not modified status - return not self.isFilled(delay) + return Utils.wait_for(lambda: self.jail.isEmpty(), delay) def assert_correct_last_attempt(self, failures, count=None): self.assertTrue(self.isFilled(20)) # give Filter a chance to react @@ -657,10 +653,11 @@ def get_monitor_failures_testcase(Filter_): self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time - self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2), + self.assertTrue(self.isEmpty(min(4, 100 * Utils.DEFAULT_SLEEP_TIME)), "Queue must be empty but it is not: %s." % (', '.join([str(x) for x in self.jail.queue]))) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, 50 * Utils.DEFAULT_SLEEP_TIME) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... @@ -685,7 +682,7 @@ def get_monitor_failures_testcase(Filter_): if interim_kill: _killfile(None, self.name) - time.sleep(0.2) # let them know + time.sleep(Utils.DEFAULT_SLEEP_TIME) # let them know # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', @@ -732,7 +729,7 @@ def get_monitor_failures_testcase(Filter_): _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) # so we should get no more failures detected - self.assertTrue(self.isEmpty(2)) + self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME)) # but then if we add it back again self.filter.addLogPath(self.name) @@ -789,19 +786,14 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover return "MonitorJournalFailures%s(%s)" \ % (Filter_, hasattr(self, 'name') and self.name or 'tempfile') - def isFilled(self, delay=2.): + def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - time0 = time.time() - while time.time() < time0 + delay: - if len(self.jail): - return True - time.sleep(0.1) - return False + return Utils.wait_for(lambda: self.jail.isFilled(), delay) - def isEmpty(self, delay=0.4): + def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): # shorter wait time for not modified status - return not self.isFilled(delay) + return Utils.wait_for(lambda: self.jail.isEmpty(), delay) def assert_correct_ban(self, test_ip, test_attempts): self.assertTrue(self.isFilled(10)) # give Filter a chance to react @@ -860,7 +852,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover _copy_lines_to_journal( self.test_file, self.journal_fields, n=5, skip=5) # so we should get no more failures detected - self.assertTrue(self.isEmpty(2)) + self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME)) # but then if we add it back again self.filter.addJournalMatch([ @@ -905,6 +897,16 @@ class GetFailures(LogCaptureTestCase): tearDownMyTime() LogCaptureTestCase.tearDown(self) + def testFilterAPI(self): + self.assertEqual(self.filter.getLogs(), []) + self.assertEqual(self.filter.getLogCount(), 0) + self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) + self.assertEqual(self.filter.getLogCount(), 1) + self.assertEqual(self.filter.getLogPaths(), [GetFailures.FILENAME_01]) + self.filter.addLogPath(GetFailures.FILENAME_02, tail=True) + self.assertEqual(self.filter.getLogCount(), 2) + self.assertEqual(sorted(self.filter.getLogPaths()), sorted([GetFailures.FILENAME_01, GetFailures.FILENAME_02])) + def testTail(self): # There must be no containters registered, otherwise [-1] indexing would be wrong self.assertEqual(self.filter.getLogs(), []) @@ -1025,6 +1027,7 @@ class GetFailures(LogCaptureTestCase): _killfile(fout, fname) def testGetFailuresUseDNS(self): + unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ('93.184.216.34', 2, 1124013539.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', @@ -1126,6 +1129,10 @@ class GetFailures(LogCaptureTestCase): class DNSUtilsTests(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + unittest.F2B.SkipIfNoNetwork() + def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertEqual(res, []) @@ -1151,6 +1158,7 @@ class DNSUtilsTests(unittest.TestCase): def testIpToName(self): res = DNSUtils.ipToName('8.8.4.4') self.assertEqual(res, 'google-public-dns-b.google.com') + unittest.F2B.SkipIfNoNetwork() # invalid ip (TEST-NET-1 according to RFC 5737) res = DNSUtils.ipToName('192.0.2.0') self.assertEqual(res, None) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index a7d7adef..d547b79a 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -36,6 +36,7 @@ from ..server.failregex import Regex, FailRegex, RegexException from ..server.server import Server from ..server.jail import Jail from ..server.jailthread import JailThread +from ..server.utils import Utils from .utils import LogCaptureTestCase from ..helpers import getLogger from .. import version @@ -74,14 +75,14 @@ class TransmitterBase(unittest.TestCase): """Call after every test case.""" self.server.quit() - def setGetTest(self, cmd, inValue, outValue=None, outCode=0, jail=None, repr_=False): + def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False): setCmd = ["set", cmd, inValue] getCmd = ["get", cmd] if jail is not None: setCmd.insert(1, jail) getCmd.insert(1, jail) - if outValue is None: + if outValue == (None,): outValue = inValue def v(x): @@ -161,15 +162,21 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) def testSleep(self): - t0 = time.time() - self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) - t1 = time.time() - # Approx 1 second delay but not faster - dt = t1 - t0 - self.assertTrue(0.99 < dt < 1.1, msg="Sleep was %g sec" % dt) + if not unittest.F2B.fast: + t0 = time.time() + self.assertEqual(self.transm.proceed(["sleep", "0.1"]), (0, None)) + t1 = time.time() + # Approx 0.1 second delay but not faster + dt = t1 - t0 + self.assertTrue(0.09 < dt < 0.2, msg="Sleep was %g sec" % dt) + else: # pragma: no cover + self.assertEqual(self.transm.proceed(["sleep", "0.0001"]), (0, None)) def testDatabase(self): - tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") + if not unittest.F2B.fast: + tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") + else: # pragma: no cover + tmpFilename = ':memory:' # Jails present, can't change database self.setGetTestNOK("dbfile", tmpFilename) self.server.delJail(self.jailName) @@ -201,8 +208,9 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed( ["set", "dbfile", "None"]), (0, None)) - os.close(tmp) - os.unlink(tmpFilename) + if not unittest.F2B.fast: + os.close(tmp) + os.unlink(tmpFilename) def testAddJail(self): jail2 = "TestJail2" @@ -225,7 +233,11 @@ class Transmitter(TransmitterBase): def testStartStopJail(self): self.assertEqual( self.transm.proceed(["start", self.jailName]), (0, None)) - time.sleep(1) + time.sleep(Utils.DEFAULT_SLEEP_TIME) + # wait until not started (3 seconds as long as any RuntimeError, ex.: RuntimeError('cannot join thread before it is started',)): + self.assertTrue( Utils.wait_for( + lambda: self.server.is_alive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), + 3) ) self.assertEqual( self.transm.proceed(["stop", self.jailName]), (0, None)) self.assertTrue(self.jailName not in self.server._Server__jails) @@ -239,9 +251,12 @@ class Transmitter(TransmitterBase): # yoh: workaround for gh-146. I still think that there is some # race condition and missing locking somewhere, but for now # giving it a small delay reliably helps to proceed with tests - time.sleep(0.1) + time.sleep(Utils.DEFAULT_SLEEP_TIME) + self.assertTrue( Utils.wait_for( + lambda: self.server.is_alive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), + 3) ) self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None)) - time.sleep(1) + self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) ) self.assertTrue(self.jailName not in self.server._Server__jails) self.assertTrue("TestJail2" not in self.server._Server__jails) @@ -297,11 +312,11 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]), (0, "127.0.0.1")) - time.sleep(1) # Give chance to ban + time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban self.assertEqual( self.transm.proceed(["set", self.jailName, "banip", "Badger"]), (0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done? - time.sleep(1) # Give chance to ban + time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban # Unban IP self.assertEqual( self.transm.proceed( diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 8eeb7b51..a9408fde 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -33,6 +33,7 @@ import unittest from .. import protocol from ..server.asyncserver import AsyncServer, AsyncServerException +from ..server.utils import Utils from ..client.csocket import CSocket @@ -54,14 +55,20 @@ class Socket(unittest.TestCase): """Test transmitter proceed method which just returns first arg""" return message + def _serverSocket(self): + try: + return CSocket(self.sock_name) + except Exception as e: + return None + def testSocket(self): serverThread = threading.Thread( target=self.server.start, args=(self.sock_name, False)) serverThread.daemon = True serverThread.start() - time.sleep(1) + time.sleep(Utils.DEFAULT_SLEEP_TIME) - client = CSocket(self.sock_name) + client = Utils.wait_for(self._serverSocket, 2) testMessage = ["A", "test", "message"] self.assertEqual(client.send(testMessage), testMessage) @@ -71,7 +78,7 @@ class Socket(unittest.TestCase): client.close() self.server.stop() - serverThread.join(1) + serverThread.join(Utils.DEFAULT_SLEEP_TIME) self.assertFalse(os.path.exists(self.sock_name)) def testSocketForce(self): @@ -85,10 +92,10 @@ class Socket(unittest.TestCase): target=self.server.start, args=(self.sock_name, True)) serverThread.daemon = True serverThread.start() - time.sleep(1) + time.sleep(Utils.DEFAULT_SLEEP_TIME) self.server.stop() - serverThread.join(1) + serverThread.join(Utils.DEFAULT_SLEEP_TIME) self.assertFalse(os.path.exists(self.sock_name)) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index dff3faae..a66325d8 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -30,8 +30,10 @@ import time import unittest from StringIO import StringIO -from ..server.mytime import MyTime from ..helpers import getLogger +from ..server.filter import DNSUtils +from ..server.mytime import MyTime +from ..server.utils import Utils logSys = getLogger(__name__) @@ -45,6 +47,43 @@ if not CONFIG_DIR: CONFIG_DIR = '/etc/fail2ban' +class F2B(): + def __init__(self, fast=False, no_network=False): + self.fast=fast + self.no_network=no_network + def SkipIfFast(self): + pass + def SkipIfNoNetwork(self): + pass + + +def initTests(opts): + if opts: # pragma: no cover + unittest.F2B = F2B(opts.fast, opts.no_network) + else: + unittest.F2B = F2B() + # --fast : + if unittest.F2B.fast: # pragma: no cover + # prevent long sleeping during test cases... + Utils.DEFAULT_SLEEP_TIME = 0.0025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.0005 + def F2B_SkipIfFast(): + raise unittest.SkipTest('Skip test because of "--fast"') + unittest.F2B.SkipIfFast = F2B_SkipIfFast + else: + # sleep intervals are large - use replacement for sleep to check time to sleep: + _org_sleep = time.sleep + def _new_sleep(v): + if (v > Utils.DEFAULT_SLEEP_TIME): + raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) + _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + time.sleep = _new_sleep + # --no-network : + if unittest.F2B.no_network: # pragma: no cover + def F2B_SkipIfNoNetwork(): + raise unittest.SkipTest('Skip test because of "--no-network"') + unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork + def mtimesleep(): # no sleep now should be necessary since polling tracks now not only # mtime but also ino and size @@ -70,7 +109,8 @@ def tearDownMyTime(): MyTime.myTime = None -def gatherTests(regexps=None, no_network=False): +def gatherTests(regexps=None, opts=None): + initTests(opts) # Import all the test cases here instead of a module level to # avoid circular imports from . import banmanagertestcase @@ -143,10 +183,10 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileFilterPoll)) - if not no_network: - tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS)) - tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) - tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) + # each test case class self will check no network, and skip it (we see it in log) + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS)) + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector @@ -165,9 +205,6 @@ def gatherTests(regexps=None, no_network=False): for file_ in os.listdir( os.path.abspath(os.path.dirname(action_d.__file__))): if file_.startswith("test_") and file_.endswith(".py"): - if no_network and file_ in ['test_badips.py','test_smtp.py']: #pragma: no cover - # Test required network - continue tests.addTest(testloader.loadTestsFromName( "%s.%s" % (action_d.__name__, os.path.splitext(file_)[0]))) @@ -182,6 +219,9 @@ def gatherTests(regexps=None, no_network=False): # yoh: Since I do not know better way for parametric tests # with good old unittest try: + # because gamin can be very slow on some platforms (and can produce many failures + # with fast sleep interval) - skip it by fast run: + unittest.F2B.SkipIfFast() from ..server.filtergamin import FilterGamin filters.append(FilterGamin) except Exception, e: # pragma: no cover @@ -276,29 +316,4 @@ class LogCaptureTestCase(unittest.TestCase): def printLog(self): print(self._log.getvalue()) -# Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid -# under cc by-sa 3.0 -if os.name == 'posix': - def pid_exists(pid): - """Check whether pid exists in the current process table.""" - import errno - if pid < 0: - return False - try: - os.kill(pid, 0) - except OSError as e: - return e.errno == errno.EPERM - else: - return True -else: - def pid_exists(pid): - import ctypes - kernel32 = ctypes.windll.kernel32 - SYNCHRONIZE = 0x100000 - - process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid) - if process != 0: - kernel32.CloseHandle(process) - return True - else: - return False +pid_exists = Utils.pid_exists From 78eb4d95eb04c6aeb8b7afa1382a6a48f9ad584a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 16 Jul 2015 17:14:10 -0400 Subject: [PATCH 129/143] RF: is_alive -> isAlive to stay consistent in camelCasing --- fail2ban/server/jail.py | 6 +++--- fail2ban/server/server.py | 8 ++++---- fail2ban/tests/dummyjail.py | 2 +- fail2ban/tests/filtertestcase.py | 2 +- fail2ban/tests/servertestcase.py | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index c5f2e9cd..cf16737a 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -229,7 +229,7 @@ class Jail: self.actions.join() logSys.info("Jail '%s' stopped" % self.name) - def is_alive(self): - """Check jail "is_alive" by checking filter and actions threads. + def isAlive(self): + """Check jail "isAlive" by checking filter and actions threads. """ - return self.filter.is_alive() or self.actions.is_alive() + return self.filter.isAlive() or self.actions.isAlive() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 1b335d02..cdfb4465 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -157,7 +157,7 @@ class Server: def startJail(self, name): try: self.__lock.acquire() - if not self.__jails[name].is_alive(): + if not self.__jails[name].isAlive(): self.__jails[name].start() finally: self.__lock.release() @@ -166,7 +166,7 @@ class Server: logSys.debug("Stopping jail %s" % name) try: self.__lock.acquire() - if self.__jails[name].is_alive(): + if self.__jails[name].isAlive(): self.__jails[name].stop() self.delJail(name) finally: @@ -323,12 +323,12 @@ class Server: def getBanTime(self, name): return self.__jails[name].actions.getBanTime() - def is_alive(self, jailnum=None): + def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 for j in self.__jails: j = self.__jails[j] - if not j.is_alive(): + if not j.isAlive(): return 0 return 1 diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index bade2f12..33a102f2 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -85,5 +85,5 @@ class DummyJail(Jail, object): def actions(self): return self.__actions; - def is_alive(self): + def isAlive(self): return True; diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 59512530..044e0346 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -594,7 +594,7 @@ def get_monitor_failures_testcase(Filter_): # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - Utils.wait_for(lambda: self.filter.is_alive(), 4*Utils.DEFAULT_SLEEP_TIME) + Utils.wait_for(lambda: self.filter.isAlive(), 4*Utils.DEFAULT_SLEEP_TIME) def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): # shorter wait time for not modified status diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index d547b79a..eedc8c94 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -236,7 +236,7 @@ class Transmitter(TransmitterBase): time.sleep(Utils.DEFAULT_SLEEP_TIME) # wait until not started (3 seconds as long as any RuntimeError, ex.: RuntimeError('cannot join thread before it is started',)): self.assertTrue( Utils.wait_for( - lambda: self.server.is_alive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), + lambda: self.server.isAlive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 3) ) self.assertEqual( self.transm.proceed(["stop", self.jailName]), (0, None)) @@ -253,7 +253,7 @@ class Transmitter(TransmitterBase): # giving it a small delay reliably helps to proceed with tests time.sleep(Utils.DEFAULT_SLEEP_TIME) self.assertTrue( Utils.wait_for( - lambda: self.server.is_alive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), + lambda: self.server.isAlive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 3) ) self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None)) self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) ) From e2a68bb02de740fb3a4d095d0af323f289e112fd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 16 Jul 2015 17:15:11 -0400 Subject: [PATCH 130/143] RF: make Jail new style class, avoiding multiple inheritance with object --- fail2ban/server/jail.py | 2 +- fail2ban/tests/dummyjail.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index cf16737a..6345d300 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -33,7 +33,7 @@ from ..helpers import getLogger logSys = getLogger(__name__) -class Jail: +class Jail(object): """Fail2Ban jail, which manages a filter and associated actions. The class handles the initialisation of a filter, and actions. It's diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 33a102f2..19f97f4e 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -28,7 +28,7 @@ from ..server.jail import Jail from ..server.actions import Actions -class DummyJail(Jail, object): +class DummyJail(Jail): """A simple 'jail' to suck in all the tickets generated by Filter's """ def __init__(self, backend=None): From 0e61b7586a5e8ca4bb769e14066af403d55d5bd0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 16 Jul 2015 17:22:36 -0400 Subject: [PATCH 131/143] RF: use descriptive loop variable (oldticket) and matching associated variable suffixes (old) --- fail2ban/server/banmanager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index a958a5c8..0ee028ef 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -263,19 +263,19 @@ class BanManager: try: self.__lock.acquire() # check already banned - for i in self.__banList: - if ticket.getIP() == i.getIP(): + for oldticket in self.__banList: + if ticket.getIP() == oldticket.getIP(): # if already permanent - btorg, torg = i.getBanTime(self.__banTime), i.getTime() - if btorg == -1: + btold, told = oldticket.getBanTime(self.__banTime), oldticket.getTime() + if btold == -1: return False # if given time is less than already banned time btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime() - if btnew != -1 and tnew + btnew <= torg + btorg: + if btnew != -1 and tnew + btnew <= told + btold: return False # we have longest ban - set new (increment) ban time - i.setTime(tnew) - i.setBanTime(btnew) + oldticket.setTime(tnew) + oldticket.setBanTime(btnew) return False # not yet banned - add new self.__banList.append(ticket) From 6faffe3201ee34d98c36c985a35f2c3810a40aec Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 17 Jul 2015 18:34:13 +0200 Subject: [PATCH 132/143] test cases extended, code review (+ python 3.x compatibility); database test cases extended - enable deleted (disabled) jail in addJail; --- MANIFEST | 1 + fail2ban/server/database.py | 8 +- fail2ban/server/filter.py | 4 +- fail2ban/server/server.py | 2 +- fail2ban/server/ticket.py | 21 ++-- fail2ban/server/utils.py | 5 +- fail2ban/tests/banmanagertestcase.py | 51 +++++++- fail2ban/tests/databasetestcase.py | 21 +++- fail2ban/tests/filtertestcase.py | 56 ++++++++- fail2ban/tests/tickettestcase.py | 176 +++++++++++++++++++++++++++ fail2ban/tests/utils.py | 4 + 11 files changed, 325 insertions(+), 24 deletions(-) create mode 100644 fail2ban/tests/tickettestcase.py diff --git a/MANIFEST b/MANIFEST index 7306cc41..fb70bb4b 100644 --- a/MANIFEST +++ b/MANIFEST @@ -331,6 +331,7 @@ fail2ban/tests/misctestcase.py fail2ban/tests/samplestestcase.py fail2ban/tests/servertestcase.py fail2ban/tests/sockettestcase.py +fail2ban/tests/tickettestcase.py fail2ban/tests/utils.py fail2ban/version.py files/bash-completion diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index c6e8c95c..3b419ed3 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -302,7 +302,7 @@ class Fail2BanDb(object): cur.execute("UPDATE jails SET enabled=0") @commitandrollback - def getJailNames(self, cur): + def getJailNames(self, cur, enabled=None): """Get name of jails in database. Currently only used for testing purposes. @@ -312,7 +312,11 @@ class Fail2BanDb(object): set Set of jail names. """ - cur.execute("SELECT name FROM jails") + if enabled is None: + cur.execute("SELECT name FROM jails") + else: + cur.execute("SELECT name FROM jails WHERE enabled=%s" % + (int(enabled),)) return set(row[0] for row in cur.fetchmany()) @commitandrollback diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 0a2a9949..e5cf648c 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -967,8 +967,8 @@ class DNSUtils: IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$") # todo: make configurable the expired time and max count of cache entries: - CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=60*60) - CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=60*60) + CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=5*60) + CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60) @staticmethod def dnsToIp(dns): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index cdfb4465..86197bc5 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -44,7 +44,7 @@ logSys = getLogger(__name__) try: from .database import Fail2BanDb -except ImportError: +except ImportError: # pragma: no cover # Dont print error here, as database may not even be used Fail2BanDb = None diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 49ebf9ea..e856b66d 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -146,16 +146,17 @@ class Ticket: # return default if not exists: if not self._data: return default - # return filtered by lambda/function: - if callable(key): - # todo: if support >= 2.7 only: - # return {k:v for k,v in self._data.iteritems() if key(k)} - return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) - # return filtered by keys: - if hasattr(key, '__iter__'): - # todo: if support >= 2.7 only: - # return {k:v for k,v in self._data.iteritems() if k in key} - return dict([(k,v) for k,v in self._data.iteritems() if k in key]) + if not isinstance(key,(str,unicode,type(None),int,float,bool,complex)): + # return filtered by lambda/function: + if callable(key): + # todo: if support >= 2.7 only: + # return {k:v for k,v in self._data.iteritems() if key(k)} + return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) + # return filtered by keys: + if hasattr(key, '__iter__'): + # todo: if support >= 2.7 only: + # return {k:v for k,v in self._data.iteritems() if k in key} + return dict([(k,v) for k,v in self._data.iteritems() if k in key]) # return single value of data: return self._data.get(key, default) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 7e69ddca..262b303d 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -52,7 +52,10 @@ class Utils(): class Cache(dict): - def __init__(self, maxCount=1000, maxTime=60*60): + def __init__(self, *args, **kwargs): + self.setOptions(*args, **kwargs) + + def setOptions(self, maxCount=1000, maxTime=60): self.maxCount = maxCount self.maxTime = maxTime diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a2d399b3..9e865a1b 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -35,27 +35,74 @@ class AddFailure(unittest.TestCase): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) def tearDown(self): """Call after every test case.""" pass def testAdd(self): + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + self.assertEqual(self.__banManager.getBanTotal(), 1) + self.__banManager.setBanTotal(0) + self.assertEqual(self.__banManager.getBanTotal(), 0) + def testAddDuplicate(self): + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) + def testAddDuplicateWithTime(self): + # add again a duplicate : + # 1) with newer start time and the same ban time + # 2) with same start time and longer ban time + # 3) with permanent ban time (-1) + for tnew, btnew in ( + (1167605999.0 + 100, None), + (1167605999.0, 24*60*60), + (1167605999.0, -1), + ): + ticket1 = BanTicket('193.168.0.128', 1167605999.0) + ticket2 = BanTicket('193.168.0.128', tnew) + if btnew is not None: + ticket2.setBanTime(btnew) + self.assertTrue(self.__banManager.addBanTicket(ticket1)) + self.assertFalse(self.__banManager.addBanTicket(ticket2)) + self.assertEqual(self.__banManager.size(), 1) + # pop ticket and check it was prolonged : + banticket = self.__banManager.getTicketByIP(ticket2.getIP()) + self.assertEqual(banticket.getTime(), ticket2.getTime()) + self.assertEqual(banticket.getTime(), ticket2.getTime()) + self.assertEqual(banticket.getBanTime(), ticket2.getBanTime(self.__banManager.getBanTime())) + def testInListOK(self): + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) def testInListNOK(self): + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) + def testUnban(self): + btime = self.__banManager.getBanTime() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager._inBanList(self.__ticket)) + self.assertEqual(self.__banManager.unBanList(self.__ticket.getTime() + btime + 1), [self.__ticket]) + self.assertEqual(self.__banManager.size(), 0) + + def testUnbanPermanent(self): + btime = self.__banManager.getBanTime() + self.__banManager.setBanTime(-1) + try: + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager._inBanList(self.__ticket)) + self.assertEqual(self.__banManager.unBanList(self.__ticket.getTime() + btime + 1), []) + self.assertEqual(self.__banManager.size(), 1) + finally: + self.__banManager.setBanTime(btime) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 20baa847..3f0e4c10 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -124,7 +124,7 @@ class DatabaseTest(LogCaptureTestCase): self.jail = DummyJail() self.db.addJail(self.jail) self.assertTrue( - self.jail.name in self.db.getJailNames(), + self.jail.name in self.db.getJailNames(True), "Jail not added to database") def testAddLog(self): @@ -332,6 +332,25 @@ class DatabaseTest(LogCaptureTestCase): actions._Actions__checkBan() self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) + def testDelAndAddJail(self): + self.testAddJail() # Add jail + # Delete jail (just disabled it): + self.db.delJail(self.jail) + jails = self.db.getJailNames() + self.assertTrue(len(jails) == 1 and self.jail.name in jails) + jails = self.db.getJailNames(enabled=False) + self.assertTrue(len(jails) == 1 and self.jail.name in jails) + jails = self.db.getJailNames(enabled=True) + self.assertTrue(len(jails) == 0) + # Add it again - should just enable it: + self.db.addJail(self.jail) + jails = self.db.getJailNames() + self.assertTrue(len(jails) == 1 and self.jail.name in jails) + jails = self.db.getJailNames(enabled=True) + self.assertTrue(len(jails) == 1 and self.jail.name in jails) + jails = self.db.getJailNames(enabled=False) + self.assertTrue(len(jails) == 0) + def testPurge(self): if Fail2BanDb is None: # pragma: no cover return diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 044e0346..df98e1ac 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -587,18 +587,18 @@ def get_monitor_failures_testcase(Filter_): def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - return Utils.wait_for(lambda: self.jail.isFilled(), delay) + return Utils.wait_for(self.jail.isFilled, delay) def _sleep_4_poll(self): # Since FilterPoll relies on time stamps and some # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - Utils.wait_for(lambda: self.filter.isAlive(), 4*Utils.DEFAULT_SLEEP_TIME) + Utils.wait_for(self.filter.isAlive, 4*Utils.DEFAULT_SLEEP_TIME) def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): # shorter wait time for not modified status - return Utils.wait_for(lambda: self.jail.isEmpty(), delay) + return Utils.wait_for(self.jail.isEmpty, delay) def assert_correct_last_attempt(self, failures, count=None): self.assertTrue(self.isFilled(20)) # give Filter a chance to react @@ -789,11 +789,11 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - return Utils.wait_for(lambda: self.jail.isFilled(), delay) + return Utils.wait_for(self.jail.isFilled, delay) def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): # shorter wait time for not modified status - return Utils.wait_for(lambda: self.jail.isEmpty(), delay) + return Utils.wait_for(self.jail.isEmpty, delay) def assert_correct_ban(self, test_ip, test_attempts): self.assertTrue(self.isFilled(10)) # give Filter a chance to react @@ -1129,6 +1129,52 @@ class GetFailures(LogCaptureTestCase): class DNSUtilsTests(unittest.TestCase): + def testCache(self): + c = Utils.Cache(maxCount=5, maxTime=60) + # not available : + self.assertTrue(c.get('a') is None) + self.assertEqual(c.get('a', 'test'), 'test') + # exact 5 elements : + for i in xrange(5): + c.set(i, i) + for i in xrange(5): + self.assertEqual(c.get(i), i) + + def testCacheMaxSize(self): + c = Utils.Cache(maxCount=5, maxTime=60) + # exact 5 elements : + for i in xrange(5): + c.set(i, i) + self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)]) + self.assertFalse(-1 in [c.get(i, -1) for i in xrange(5)]) + # add one - too many: + c.set(10, i) + # one element should be removed : + self.assertTrue(-1 in [c.get(i, -1) for i in xrange(5)]) + # test max size (not expired): + for i in xrange(10): + c.set(i, 1) + self.assertEqual(len(c), 5) + + def testCacheMaxTime(self): + # test max time (expired, timeout reached) : + c = Utils.Cache(maxCount=5, maxTime=0.0005) + for i in xrange(10): + c.set(i, 1) + st = time.time() + self.assertTrue(Utils.wait_for(lambda: time.time() >= st + 0.0005, 1)) + # we have still 5 elements (or fewer if too slow test mashine): + self.assertTrue(len(c) <= 5) + # but all that are expiered also: + for i in xrange(10): + self.assertTrue(c.get(i) is None) + # here the whole cache should be empty: + self.assertEqual(len(c), 0) + + + +class DNSUtilsNetworkTests(unittest.TestCase): + def setUp(self): """Call before every test case.""" unittest.F2B.SkipIfNoNetwork() diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py new file mode 100644 index 00000000..68a44bb5 --- /dev/null +++ b/fail2ban/tests/tickettestcase.py @@ -0,0 +1,176 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +__author__ = "Serg G. Brester (sebres)" +__copyright__ = "Copyright (c) 2015 Serg G. Brester, 2015- Fail2Ban Contributors" +__license__ = "GPL" + +from ..server.mytime import MyTime +import unittest + +from ..server.ticket import Ticket, FailTicket, BanTicket + + +class TicketTests(unittest.TestCase): + + def testTicket(self): + + tm = MyTime.time() + matches = ['first', 'second'] + matches2 = ['first', 'second'] + matches3 = ['first', 'second', 'third'] + + # Ticket + t = Ticket('193.168.0.128', tm, matches) + self.assertEqual(t.getIP(), '193.168.0.128') + self.assertEqual(t.getTime(), tm) + self.assertEqual(t.getMatches(), matches2) + t.setAttempt(2) + self.assertEqual(t.getAttempt(), 2) + t.setBanCount(10) + self.assertEqual(t.getBanCount(), 10) + # default ban time (from manager): + self.assertEqual(t.getBanTime(60*60), 60*60) + self.assertFalse(t.isTimedOut(tm + 60 + 1, 60*60)) + self.assertTrue(t.isTimedOut(tm + 60*60 + 1, 60*60)) + t.setBanTime(60) + self.assertEqual(t.getBanTime(60*60), 60) + self.assertEqual(t.getBanTime(), 60) + self.assertFalse(t.isTimedOut(tm)) + self.assertTrue(t.isTimedOut(tm + 60 + 1)) + # permanent : + t.setBanTime(-1) + self.assertFalse(t.isTimedOut(tm + 60 + 1)) + t.setBanTime(60) + + # BanTicket + tm = MyTime.time() + matches = ['first', 'second'] + ft = FailTicket('193.168.0.128', tm, matches) + ft.setBanTime(60*60) + self.assertEqual(ft.getIP(), '193.168.0.128') + self.assertEqual(ft.getTime(), tm) + self.assertEqual(ft.getMatches(), matches2) + ft.setAttempt(2) + self.assertEqual(ft.getAttempt(), 2) + # retry is max of set retry and failures: + self.assertEqual(ft.getRetry(), 2) + ft.setRetry(1) + self.assertEqual(ft.getRetry(), 2) + ft.setRetry(3) + self.assertEqual(ft.getRetry(), 3) + ft.inc() + self.assertEqual(ft.getAttempt(), 3) + self.assertEqual(ft.getRetry(), 4) + self.assertEqual(ft.getMatches(), matches2) + # with 1 match, 1 failure and factor 10 (retry count) : + ft.inc(['third'], 1, 10) + self.assertEqual(ft.getAttempt(), 4) + self.assertEqual(ft.getRetry(), 14) + self.assertEqual(ft.getMatches(), matches3) + # last time (ignore if smaller as time): + self.assertEqual(ft.getLastTime(), tm) + ft.setLastTime(tm-60) + self.assertEqual(ft.getTime(), tm) + self.assertEqual(ft.getLastTime(), tm) + ft.setLastTime(tm+60) + self.assertEqual(ft.getTime(), tm+60) + self.assertEqual(ft.getLastTime(), tm+60) + ft.setData('country', 'DE') + self.assertEqual(ft.getData(), + {'matches': ['first', 'second', 'third'], 'failures': 4, 'country': 'DE'}) + + # copy all from another ticket: + ft2 = FailTicket(ticket=ft) + self.assertEqual(ft, ft2) + self.assertEqual(ft.getData(), ft2.getData()) + self.assertEqual(ft2.getAttempt(), 4) + self.assertEqual(ft2.getRetry(), 14) + self.assertEqual(ft2.getMatches(), matches3) + self.assertEqual(ft2.getTime(), ft.getTime()) + self.assertEqual(ft2.getLastTime(), ft.getLastTime()) + self.assertEqual(ft2.getBanTime(), ft.getBanTime()) + + def testTicketData(self): + t = BanTicket('193.168.0.128', None, ['first', 'second']) + # expand data (no overwrites, matches are available) : + t.setData('region', 'Hamburg', 'country', 'DE', 'city', 'Hamburg') + self.assertEqual( + t.getData(), + {'matches': ['first', 'second'], 'failures':0, 'region': 'Hamburg', 'country': 'DE', 'city': 'Hamburg'}) + # at once as dict (single argument, overwrites it completelly, no more matches/failures) : + t.setData({'region': None, 'country': 'FR', 'city': 'Paris'},) + self.assertEqual( + t.getData(), + {'city': 'Paris', 'country': 'FR'}) + # at once as dict (overwrites it completelly, no more matches/failures) : + t.setData({'region': 'Hamburg', 'country': 'DE', 'city': None}) + self.assertEqual( + t.getData(), + {'region': 'Hamburg', 'country': 'DE'}) + self.assertEqual( + t.getData('region'), + 'Hamburg') + self.assertEqual( + t.getData('country'), + 'DE') + # again, named arguments: + t.setData(region='Bremen', city='Bremen') + self.assertEqual(t.getData(), + {'region': 'Bremen', 'country': 'DE', 'city': 'Bremen'}) + # again, but as args (key value pair): + t.setData('region', 'Brandenburg', 'city', 'Berlin') + self.assertEqual( + t.getData('region'), + 'Brandenburg') + self.assertEqual( + t.getData('city'), + 'Berlin') + self.assertEqual( + t.getData(), + {'city':'Berlin', 'region': 'Brandenburg', 'country': 'DE'}) + # interator filter : + self.assertEqual( + t.getData(('city', 'country')), + {'city':'Berlin', 'country': 'DE'}) + # callable filter : + self.assertEqual( + t.getData(lambda k: k.upper() == 'COUNTRY'), + {'country': 'DE'}) + # remove one data entry: + t.setData('city', None) + self.assertEqual( + t.getData(), + {'region': 'Brandenburg', 'country': 'DE'}) + # default if not available: + self.assertEqual( + t.getData('city', 'Unknown'), + 'Unknown') + # add continent : + t.setData('continent', 'Europe') + # again, but as argument list (overwrite new only, leave continent unchanged) : + t.setData(*['country', 'RU', 'region', 'Moscow']) + self.assertEqual( + t.getData(), + {'continent': 'Europe', 'country': 'RU', 'region': 'Moscow'}) + # clear: + t.setData({}) + self.assertEqual(t.getData(), {}) + self.assertEqual(t.getData('anything', 'default'), 'default') diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index a66325d8..017ad3f6 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -115,6 +115,7 @@ def gatherTests(regexps=None, opts=None): # avoid circular imports from . import banmanagertestcase from . import clientreadertestcase + from . import tickettestcase from . import failmanagertestcase from . import filtertestcase from . import servertestcase @@ -150,6 +151,8 @@ def gatherTests(regexps=None, opts=None): tests.addTest(unittest.makeSuite(servertestcase.LoggingTests)) tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest)) tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions)) + # Ticket, BanTicket, FailTicket + tests.addTest(unittest.makeSuite(tickettestcase.TicketTests)) # FailManager tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager @@ -187,6 +190,7 @@ def gatherTests(regexps=None, opts=None): tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS)) tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsNetworkTests)) tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector From cf3cf27fa3c57c6a43448e23faa2b37c553395ee Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 17 Jul 2015 21:05:50 +0200 Subject: [PATCH 133/143] - filtering of test cases extended for method additionally to class (matches method only not whole class for regexp matched method name); - new options for "fail2ban-testcases" introduced: "-g" or "--no-gamin" and "-m" or "--memory-db", both are true also if "-f" or "--fast" specified, for example: `fail2ban-testcases -ngm` will runs faster (because no network, no gamin, memory database) but will use the same default sleep intervals as in production (in comparison to -nf); - seekToTime rewritten, accuracy increased by seekToTime, extended for all FileFilter (PyInotify, Gamin also), test cases extended etc. - common performance optimized and code reviewed; --- bin/fail2ban-testcases | 6 + fail2ban/server/filter.py | 119 +++++++++------ fail2ban/server/filterpoll.py | 7 +- fail2ban/server/ticket.py | 6 + fail2ban/tests/databasetestcase.py | 26 ++-- fail2ban/tests/filtertestcase.py | 228 +++++++++++++++++++++-------- fail2ban/tests/servertestcase.py | 7 +- fail2ban/tests/utils.py | 45 +++--- 8 files changed, 301 insertions(+), 143 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 28af8265..22a44a8b 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -58,6 +58,12 @@ def get_opt_parser(): Option('-n', "--no-network", action="store_true", dest="no_network", help="Do not run tests that require the network"), + Option('-g', "--no-gamin", action="store_true", + dest="no_gamin", + help="Do not run tests that require the gamin"), + Option('-m', "--memory-db", action="store_true", + dest="memory_db", + help="Run database tests using memory instead of file"), Option('-f', "--fast", action="store_true", dest="fast", help="Try to increase speed of the tests, decreasing of wait intervals, memory database"), diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e5cf648c..d1ac3680 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -560,6 +560,7 @@ class FileFilter(Filter): Filter.__init__(self, jail, **kwargs) ## The log file path. self.__logs = dict() + self.__autoSeek = dict() self.setLogEncoding("auto") ## @@ -567,7 +568,7 @@ class FileFilter(Filter): # # @param path log file path - def addLogPath(self, path, tail=False): + def addLogPath(self, path, tail=False, autoSeek=True): if path in self.__logs: logSys.error(path + " already exists") else: @@ -579,6 +580,11 @@ class FileFilter(Filter): log.setPos(lastpos) self.__logs[path] = log logSys.info("Added logfile = %s (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) + if autoSeek: + # if default, seek to "current time" - "find time": + if isinstance(autoSeek, bool): + autoSeek = MyTime.time() - self.getFindTime() + self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific def _addLogPath(self, path): @@ -673,7 +679,7 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename, startTime=None): + def getFailures(self, filename): log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in " + filename) @@ -695,13 +701,17 @@ class FileFilter(Filter): logSys.exception(e) return False - # prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm: - if log.getPos() == 0 and startTime is not None: + # seek to find time for first usage only (prevent performance decline with polling of big files) + if self.__autoSeek.get(filename): + startTime = self.__autoSeek[filename] + del self.__autoSeek[filename] + # prevent completely read of big files first time (after start of service), + # initial seek to start time using half-interval search algorithm: try: - # startTime = MyTime.time() - self.getFindTime() self.seekToTime(log, startTime) except Exception, e: # pragma: no cover logSys.error("Error during seek to start time in \"%s\"", filename) + raise logSys.exception(e) return False @@ -726,71 +736,88 @@ class FileFilter(Filter): # Seeks to line with date (search using half-interval search algorithm), to start polling from it # - def seekToTime(self, container, date): + def seekToTime(self, container, date, accuracy=3): fs = container.getFileSize() if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Seek to find time %s (%s), file size %s", date, datetime.datetime.fromtimestamp(date).strftime("%Y-%m-%d %H:%M:%S"), fs) - date -= 0.009 - minp = 0 + minp = container.getPos() maxp = fs - lastpos = 0 - lastFew = 0 - lastTime = None + tryPos = minp + lastPos = -1 + foundPos = 0 + foundTime = None cntr = 0 unixTime = None - lasti = 0 - movecntr = 1 + movecntr = accuracy while maxp > minp: - i = int(minp + (maxp - minp) / 2) - pos = container.seek(i) + if tryPos is None: + pos = int(minp + (maxp - minp) / 2) + else: + pos, tryPos = tryPos, None + # because container seek will go to start of next line (minus CRLF): + pos = max(0, pos-2) + seekpos = pos = container.seek(pos) cntr += 1 # within next 5 lines try to find any legal datetime: lncntr = 5; dateTimeMatch = None - llen = 0 - if lastpos == pos: - i = pos + nextp = None while True: line = container.readline() if not line: break - llen += len(line) - l = line.rstrip('\r\n') - (timeMatch, template) = self.dateDetector.matchTime(l) + (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: - dateTimeMatch = self.dateDetector.getTime2(l[timeMatch.start():timeMatch.end()], (timeMatch, template)) + dateTimeMatch = self.dateDetector.getTime2(line[timeMatch.start():timeMatch.end()], (timeMatch, template)) + else: + nextp = container.tell() + if nextp > maxp: + pos = seekpos + break + pos = nextp if not dateTimeMatch and lncntr: lncntr -= 1 continue break + # not found at this step - stop searching + if dateTimeMatch: + unixTime = dateTimeMatch[0] + if unixTime >= date: + if foundTime is None or unixTime <= foundTime: + foundPos = pos + foundTime = unixTime + if pos == maxp: + pos = seekpos + if pos < maxp: + maxp = pos + else: + if foundTime is None or unixTime >= foundTime: + foundPos = pos + foundTime = unixTime + if nextp is None: + nextp = container.tell() + pos = nextp + if pos > minp: + minp = pos # if we can't move (position not changed) - if i + llen == lasti: + if pos == lastPos: movecntr -= 1 if movecntr <= 0: break - lasti = i + llen; - # not found at this step - stop searching - if not dateTimeMatch: + # we have found large area without any date mached + # or end of search - try min position (because can be end of previous line): + if minp != lastPos: + lastPos = tryPos = minp + continue break - unixTime = dateTimeMatch[0] - if unixTime >= date: - maxp = i - else: - minp = i + llen - lastFew = pos; - lastTime = unixTime - lastpos = pos - # if found position have a time greater as given - use smallest time we have found - if unixTime is None or unixTime > date: - unixTime = lastTime - lastpos = container.seek(lastFew, False) - else: - lastpos = container.seek(lastpos, False) - container.setPos(lastpos) + lastPos = pos + # always use smallest pos, that could be found: + foundPos = container.seek(minp, False) + container.setPos(foundPos) if logSys.getEffectiveLevel() <= logging.DEBUG: - logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, - (datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr) + logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastPos, fs, foundTime, + (datetime.datetime.fromtimestamp(foundTime).strftime("%Y-%m-%d %H:%M:%S") if foundTime is not None else ''), cntr) def status(self, flavor="basic"): """Status of Filter plus files being monitored. @@ -898,11 +925,15 @@ class FileContainer: # seek to given position h.seek(offs, 0) # goto end of next line - if endLine: + if offs and endLine: h.readline() # get current real position return h.tell() + def tell(self): + # get current real position + return self.__handler.tell() + @staticmethod def decode_line(filename, enc, line): try: diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 1afda7f2..d7ff9075 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -58,7 +58,6 @@ class FilterPoll(FileFilter): ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() - self.__initial = dict() logSys.debug("Created FilterPoll") ## @@ -109,11 +108,7 @@ class FilterPoll(FileFilter): modlst = [] Utils.wait_for(lambda: self.getModified(modlst), self.sleeptime) for filename in modlst: - # set start time as now - find time for first usage only (prevent performance bug with polling of big files) - self.getFailures(filename, - (MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None - ) - self.__initial[filename] = True + self.getFailures(filename) self.__modified = True if self.__modified: diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index e856b66d..b3eea052 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -176,6 +176,9 @@ class FailTicket(Ticket): self.__retry = self._data['failures']; def setRetry(self, value): + """ Set artificial retry count, normally equal failures / attempt, + used in incremental features (BanTimeIncr) to increase retry count for bad IPs + """ self.__retry = value if not self._data['failures']: self._data['failures'] = 1 @@ -184,6 +187,9 @@ class FailTicket(Ticket): self._data['matches'] = [] def getRetry(self): + """ Returns failures / attempt count or + artificial retry count increased for bad IPs + """ return max(self.__retry, self._data['failures']) def inc(self, matches=None, attempt=1, count=1): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 3f0e4c10..6b85af56 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -36,18 +36,20 @@ from ..server.actions import Actions from .dummyjail import DummyJail try: from ..server.database import Fail2BanDb as Fail2BanDb - # because of tests performance use memory instead of file: - def TestFail2BanDb(filename): - if unittest.F2B.fast: - return Fail2BanDb(':memory:') - return Fail2BanDb(filename) -except ImportError: +except ImportError: # pragma: no cover Fail2BanDb = None from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") +# because of tests performance use memory instead of file: +def getFail2BanDb(filename): + if unittest.F2B.memory_db: # pragma: no cover + return Fail2BanDb(':memory:') + return Fail2BanDb(filename) + + class DatabaseTest(LogCaptureTestCase): def setUp(self): @@ -59,8 +61,10 @@ class DatabaseTest(LogCaptureTestCase): "available.") elif Fail2BanDb is None: return - _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") - self.db = TestFail2BanDb(self.dbFilename) + self.dbFilename = None + if not unittest.F2B.memory_db: + _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") + self.db = getFail2BanDb(self.dbFilename) def tearDown(self): """Call after every test case.""" @@ -68,7 +72,8 @@ class DatabaseTest(LogCaptureTestCase): if Fail2BanDb is None: # pragma: no cover return # Cleanup - os.remove(self.dbFilename) + if self.dbFilename is not None: + os.remove(self.dbFilename) def testGetFilename(self): if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover @@ -106,6 +111,9 @@ class DatabaseTest(LogCaptureTestCase): def testUpdateDb(self): if Fail2BanDb is None: # pragma: no cover return + self.db = None + if self.dbFilename is None: # pragma: no cover + _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") shutil.copyfile( os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename) self.db = Fail2BanDb(self.dbFilename) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index df98e1ac..11642d2c 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -81,6 +81,16 @@ def _killfile(f, name): _killfile(None, name + '.bak') +def _maxWaitTime(wtime): + if unittest.F2B.fast: + wtime /= 10 + return wtime + + +def _tm(time): + return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") + + def _assert_equal_entries(utest, found, output, count=None): """Little helper to unify comparisons with the target entries @@ -122,9 +132,15 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): Test filter to contain target ticket """ if isinstance(filter_, DummyJail): + # get fail ticket from jail found = _ticket_tuple(filter_.getFailTicket()) else: # when we are testing without jails + # wait for failures (up to max time) + Utils.wait_for( + lambda: filter_.failManager.getFailTotal() >= (count if count else output[1]), + _maxWaitTime(10)) + # get fail ticket from filter found = _ticket_tuple(filter_.failManager.toBan()) _assert_equal_entries(utest, found, output, count) @@ -352,59 +368,132 @@ class LogFileFilterPoll(unittest.TestCase): self.assertTrue(self.filter.isModified(LogFileFilterPoll.FILENAME)) self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME)) - def testSeekToTime(self): + def testSeekToTimeSmallFile(self): fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') - tm = lambda time: datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") time = 1417512352 f = open(fname, 'w') - fc = FileContainer(fname, self.filter.getLogEncoding()) - fc.open() - fc.setPos(0); self.filter.seekToTime(fc, time) + fc = None try: + fc = FileContainer(fname, self.filter.getLogEncoding()) + fc.open() + fc.setPos(0); self.filter.seekToTime(fc, time) f.flush() # empty : fc.setPos(0); self.filter.seekToTime(fc, time) self.assertEqual(fc.getPos(), 0) # one entry with exact time: - f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time)) + f.write("%s [sshd] error: PAM: failure len 1\n" % _tm(time)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) - # one entry with smaller time: + + # rewrite : f.seek(0) - f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time - 10)) + f.truncate() + fc.close() + fc = FileContainer(fname, self.filter.getLogEncoding()) + fc.open() + # no time - nothing should be found : + for i in xrange(10): + f.write("[sshd] error: PAM: failure len 1\n") + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + + # rewrite + f.seek(0) + f.truncate() + fc.close() + fc = FileContainer(fname, self.filter.getLogEncoding()) + fc.open() + # one entry with smaller time: + f.write("%s [sshd] error: PAM: failure len 2\n" % _tm(time - 10)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) - self.assertEqual(fc.getPos(), 0) - f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time - 9)) - f.flush() - fc.setPos(0); self.filter.seekToTime(fc, time) - self.assertEqual(fc.getPos(), 0) - # add exact time between: - f.write("%s [sshd] error: PAM: failure\n" % tm(time - 1)) + self.assertEqual(fc.getPos(), 53) + # two entries with smaller time: + f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % _tm(time - 9)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) self.assertEqual(fc.getPos(), 110) + # check move after end (all of time smaller): + f.write("%s [sshd] error: PAM: failure\n" % _tm(time - 1)) + f.flush() + self.assertEqual(fc.getFileSize(), 157) + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 157) + # stil one exact line: - f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time)) - f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time)) + f.write("%s [sshd] error: PAM: Authentication failure\n" % _tm(time)) + f.write("%s [sshd] error: PAM: failure len 1\n" % _tm(time)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) - self.assertEqual(fc.getPos(), 110) + self.assertEqual(fc.getPos(), 157) + # add something hereafter: - f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 2)) - f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time + 3)) + f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % _tm(time + 2)) + f.write("%s [sshd] error: PAM: Authentication failure\n" % _tm(time + 3)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) - self.assertEqual(fc.getPos(), 110) + self.assertEqual(fc.getPos(), 157) # add something hereafter: - f.write("%s [sshd] error: PAM: failure\n" % tm(time + 9)) - f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 9)) + f.write("%s [sshd] error: PAM: failure\n" % _tm(time + 9)) + f.write("%s [sshd] error: PAM: failure len 4 3 2\n" % _tm(time + 9)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) - self.assertEqual(fc.getPos(), 110) + self.assertEqual(fc.getPos(), 157) + # start search from current pos : + fc.setPos(157); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 157) + # start search from current pos : + fc.setPos(110); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 157) finally: - fc.close() + if fc: + fc.close() + _killfile(f, fname) + + def testSeekToTimeLargeFile(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') + time = 1417512352 + f = open(fname, 'w') + fc = None + count = 1000 if unittest.F2B.fast else 10000 + try: + fc = FileContainer(fname, self.filter.getLogEncoding()) + fc.open() + f.seek(0) + # variable length of file (ca 45K or 450K before and hereafter): + # write lines with smaller as search time: + t = time - count - 1 + for i in xrange(count): + f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) + t += 1 + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 47*count) + # write lines with exact search time: + for i in xrange(10): + f.write("%s [sshd] error: PAM: failure\n" % _tm(time)) + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 47*count) + fc.setPos(4*count); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 47*count) + # write lines with greater as search time: + t = time+1 + for i in xrange(count//500): + for j in xrange(500): + f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) + t += 1 + f.flush() + fc.setPos(0); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 47*count) + fc.setPos(53); self.filter.seekToTime(fc, time) + self.assertEqual(fc.getPos(), 47*count) + + finally: + if fc: + fc.close() _killfile(f, fname) class LogFileMonitor(LogCaptureTestCase): @@ -418,7 +507,7 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) - self.filter.addLogPath(self.name) + self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -431,11 +520,12 @@ class LogFileMonitor(LogCaptureTestCase): def isModified(self, delay=2.): """Wait up to `delay` sec to assure that it was modified or not """ - return Utils.wait_for(lambda: self.filter.isModified(self.name), delay) + return Utils.wait_for(lambda: self.filter.isModified(self.name), _maxWaitTime(delay)) - def notModified(self): - # shorter wait time for not modified status - return not self.isModified(4*Utils.DEFAULT_SLEEP_TIME) + def notModified(self, delay=2.): + """Wait up to `delay` sec as long as it was not modified + """ + return Utils.wait_for(lambda: not self.filter.isModified(self.name), _maxWaitTime(delay)) def testUnaccessibleLogFile(self): os.chmod(self.name, 0) @@ -478,7 +568,7 @@ class LogFileMonitor(LogCaptureTestCase): os.rename(self.name, self.name + '.old') # we are not signaling as modified whenever # it gets away - self.assertTrue(self.notModified()) + self.assertTrue(self.notModified(1)) f = open(self.name, 'a') self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) @@ -562,7 +652,7 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) - self.filter.addLogPath(self.name) + self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") self.filter.start() @@ -587,21 +677,21 @@ def get_monitor_failures_testcase(Filter_): def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - return Utils.wait_for(self.jail.isFilled, delay) + return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay)) def _sleep_4_poll(self): # Since FilterPoll relies on time stamps and some # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - Utils.wait_for(self.filter.isAlive, 4*Utils.DEFAULT_SLEEP_TIME) + Utils.wait_for(self.filter.isAlive, _maxWaitTime(5)) - def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): + def isEmpty(self, delay=_maxWaitTime(5)): # shorter wait time for not modified status - return Utils.wait_for(self.jail.isEmpty, delay) + return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay)) def assert_correct_last_attempt(self, failures, count=None): - self.assertTrue(self.isFilled(20)) # give Filter a chance to react + self.assertTrue(self.isFilled(10)) # give Filter a chance to react _assert_correct_last_attempt(self, self.jail, failures, count=count) def test_grow_file(self): @@ -616,7 +706,7 @@ def get_monitor_failures_testcase(Filter_): # since it should have not been enough _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) - self.assertTrue(self.isFilled(6)) + self.assertTrue(self.isFilled(10)) # so we sleep for up to 2 sec for it not to become empty, # and meanwhile pass to other thread(s) and filter should # have gathered new failures and passed them into the @@ -653,11 +743,11 @@ def get_monitor_failures_testcase(Filter_): self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time - self.assertTrue(self.isEmpty(min(4, 100 * Utils.DEFAULT_SLEEP_TIME)), + self.assertTrue(self.isEmpty(_maxWaitTime(5)), "Queue must be empty but it is not: %s." % (', '.join([str(x) for x in self.jail.queue]))) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) - Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, 50 * Utils.DEFAULT_SLEEP_TIME) + Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... @@ -682,7 +772,7 @@ def get_monitor_failures_testcase(Filter_): if interim_kill: _killfile(None, self.name) - time.sleep(Utils.DEFAULT_SLEEP_TIME) # let them know + time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) # let them know # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', @@ -729,10 +819,10 @@ def get_monitor_failures_testcase(Filter_): _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) # so we should get no more failures detected - self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME)) + self.assertTrue(self.isEmpty(_maxWaitTime(10))) - # but then if we add it back again - self.filter.addLogPath(self.name) + # but then if we add it back again (no seek to time in FileFilter's, because in file used the same time) + self.filter.addLogPath(self.name, autoSeek=False) # Tricky catch here is that it should get them from the # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) @@ -789,14 +879,14 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def isFilled(self, delay=1.): """Wait up to `delay` sec to assure that it was modified or not """ - return Utils.wait_for(self.jail.isFilled, delay) + return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay)) - def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME): + def isEmpty(self, delay=_maxWaitTime(5)): # shorter wait time for not modified status - return Utils.wait_for(self.jail.isEmpty, delay) + return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay)) def assert_correct_ban(self, test_ip, test_attempts): - self.assertTrue(self.isFilled(10)) # give Filter a chance to react + self.assertTrue(self.isFilled(_maxWaitTime(10))) # give Filter a chance to react ticket = self.jail.getFailTicket() attempts = ticket.getAttempt() @@ -819,7 +909,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover _copy_lines_to_journal( self.test_file, self.journal_fields, skip=2, n=3) - self.assertTrue(self.isFilled(6)) + self.assertTrue(self.isFilled(10)) # so we sleep for up to 6 sec for it not to become empty, # and meanwhile pass to other thread(s) and filter should # have gathered new failures and passed them into the @@ -852,7 +942,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover _copy_lines_to_journal( self.test_file, self.journal_fields, n=5, skip=5) # so we should get no more failures detected - self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME)) + self.assertTrue(self.isEmpty(_maxWaitTime(10))) # but then if we add it back again self.filter.addJournalMatch([ @@ -863,7 +953,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures - self.assertTrue(self.isFilled(6)) + self.assertTrue(self.isFilled(10)) return MonitorJournalFailures @@ -931,7 +1021,7 @@ class GetFailures(LogCaptureTestCase): filename = filename or GetFailures.FILENAME_01 failures = failures or GetFailures.FAILURES_01 - self.filter.addLogPath(filename) + self.filter.addLogPath(filename, autoSeek=0) self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) $") self.filter.getFailures(filename) _assert_correct_last_attempt(self, self.filter, failures) @@ -955,7 +1045,7 @@ class GetFailures(LogCaptureTestCase): [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) - self.filter.addLogPath(GetFailures.FILENAME_02) + self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) self.filter.addFailRegex("Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) @@ -963,25 +1053,35 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03(self): output = ('203.162.223.135', 7, 1124013544.0) - self.filter.addLogPath(GetFailures.FILENAME_03) + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex("error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) - def testGetFailures03_seek(self): + def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... output = ('203.162.223.135', 5, 1124013544.0) - self.filter.addLogPath(GetFailures.FILENAME_03) + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) self.filter.addFailRegex("error,relay=,.*550 User unknown") - self.filter.getFailures(GetFailures.FILENAME_03, output[2] - 4*60 + 1) + self.filter.getFailures(GetFailures.FILENAME_03) + _assert_correct_last_attempt(self, self.filter, output) + + def testGetFailures03_Seek2(self): + # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... + output = ('203.162.223.135', 1, 1124013544.0) + self.filter.setMaxRetry(1) + + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) + self.filter.addFailRegex("error,relay=,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) def testGetFailures04(self): output = [('212.41.96.186', 4, 1124013600.0), ('212.41.96.185', 4, 1124017198.0)] - self.filter.addLogPath(GetFailures.FILENAME_04) + self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0) self.filter.addFailRegex("Invalid user .* ") self.filter.getFailures(GetFailures.FILENAME_04) @@ -1048,7 +1148,7 @@ class GetFailures(LogCaptureTestCase): filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures - filter_.addLogPath(GetFailures.FILENAME_USEDNS) + filter_.addLogPath(GetFailures.FILENAME_USEDNS, autoSeek=False) filter_.addFailRegex("Failed .* from ") filter_.getFailures(GetFailures.FILENAME_USEDNS) _assert_correct_last_attempt(self, filter_, output) @@ -1056,14 +1156,14 @@ class GetFailures(LogCaptureTestCase): def testGetFailuresMultiRegex(self): output = ('141.3.81.106', 8, 1124013541.0) - self.filter.addLogPath(GetFailures.FILENAME_02) + self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex("Failed .* from ") self.filter.addFailRegex("Accepted .* from ") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresIgnoreRegex(self): - self.filter.addLogPath(GetFailures.FILENAME_02) + self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex("Failed .* from ") self.filter.addFailRegex("Accepted .* from ") self.filter.addIgnoreRegex("for roehl") @@ -1075,7 +1175,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailuresMultiLine(self): output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] - self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) self.filter.setMaxRetry(1) @@ -1093,7 +1193,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailuresMultiLineIgnoreRegex(self): output = [("192.0.43.10", 2, 1124013599.0)] - self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") self.filter.setMaxLines(100) @@ -1109,7 +1209,7 @@ class GetFailures(LogCaptureTestCase): output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), ("192.0.43.15", 1, 1124013598.0)] - self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") self.filter.setMaxLines(100) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index eedc8c94..d26d7562 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -76,12 +76,15 @@ class TransmitterBase(unittest.TestCase): self.server.quit() def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False): + """Process set/get commands and compare both return values + with outValue if it was given otherwise with inValue""" setCmd = ["set", cmd, inValue] getCmd = ["get", cmd] if jail is not None: setCmd.insert(1, jail) getCmd.insert(1, jail) + # if outValue was not given (now None is allowed return/compare value also) if outValue == (None,): outValue = inValue @@ -173,7 +176,7 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed(["sleep", "0.0001"]), (0, None)) def testDatabase(self): - if not unittest.F2B.fast: + if not unittest.F2B.memory_db: tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") else: # pragma: no cover tmpFilename = ':memory:' @@ -208,7 +211,7 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed( ["set", "dbfile", "None"]), (0, None)) - if not unittest.F2B.fast: + if not unittest.F2B.memory_db: os.close(tmp) os.unlink(tmpFilename) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 017ad3f6..338ccec1 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" import logging +import optparse import os import re import sys @@ -35,6 +36,7 @@ from ..server.filter import DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils + logSys = getLogger(__name__) CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None) @@ -47,10 +49,14 @@ if not CONFIG_DIR: CONFIG_DIR = '/etc/fail2ban' -class F2B(): - def __init__(self, fast=False, no_network=False): - self.fast=fast - self.no_network=no_network +class F2B(optparse.Values): + def __init__(self, opts={}): + self.__dict__ = opts.__dict__ if opts else { + 'fast': False, 'memory_db':False, 'no_gamin': False, 'no_network': False, + } + if self.fast: + self.memory_db = True + self.no_gamin = True def SkipIfFast(self): pass def SkipIfNoNetwork(self): @@ -58,13 +64,11 @@ class F2B(): def initTests(opts): - if opts: # pragma: no cover - unittest.F2B = F2B(opts.fast, opts.no_network) - else: - unittest.F2B = F2B() + unittest.F2B = F2B(opts) # --fast : if unittest.F2B.fast: # pragma: no cover - # prevent long sleeping during test cases... + # racy decrease default sleep intervals to test it faster + # (prevent long sleeping during test cases ... less time goes to sleep): Utils.DEFAULT_SLEEP_TIME = 0.0025 Utils.DEFAULT_SLEEP_INTERVAL = 0.0005 def F2B_SkipIfFast(): @@ -74,7 +78,7 @@ def initTests(opts): # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if (v > Utils.DEFAULT_SLEEP_TIME): + if (v > Utils.DEFAULT_SLEEP_TIME): # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) time.sleep = _new_sleep @@ -103,7 +107,7 @@ def setUpMyTime(): def tearDownMyTime(): os.environ.pop('TZ') - if old_TZ: + if old_TZ: # pragma: no cover os.environ['TZ'] = old_TZ time.tzset() MyTime.myTime = None @@ -135,11 +139,15 @@ def gatherTests(regexps=None, opts=None): _regexps = [re.compile(r) for r in regexps] def addTest(self, suite): - suite_str = str(suite) - for r in self._regexps: - if r.search(suite_str): - super(FilteredTestSuite, self).addTest(suite) - return + matched = [] + for test in suite: + s = str(test) + for r in self._regexps: + if r.search(s): + matched.append(test) + break + for test in matched: + super(FilteredTestSuite, self).addTest(test) tests = FilteredTestSuite() @@ -160,7 +168,7 @@ def gatherTests(regexps=None, opts=None): try: import dns tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) - except ImportError: + except ImportError: # pragma: no cover pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) @@ -225,7 +233,8 @@ def gatherTests(regexps=None, opts=None): try: # because gamin can be very slow on some platforms (and can produce many failures # with fast sleep interval) - skip it by fast run: - unittest.F2B.SkipIfFast() + if unittest.F2B.fast or unittest.F2B.no_gamin: # pragma: no cover + raise Exception('Skip, fast: %s, no_gamin: %s' % (unittest.F2B.fast, unittest.F2B.no_gamin)) from ..server.filtergamin import FilterGamin filters.append(FilterGamin) except Exception, e: # pragma: no cover From 770c219ab69411029910b4ee2d9cb8bc7b25bccc Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 22 Jul 2015 17:32:37 +0200 Subject: [PATCH 134/143] SetupTest should run at version of python currently installed (resp. active one), skip if started with another version. --- fail2ban/tests/misctestcase.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index a2310ec5..c4dd716d 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -66,6 +66,17 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitcommaspace(' 1, 2 , '), ['1', '2']) +def _getSysPythonVersion(): + import subprocess, locale + sysVerCmd = "python -c 'import sys; print(tuple(sys.version_info))'" + if sys.version_info >= (2,7): + sysVer = subprocess.check_output(sysVerCmd, shell=True) + else: + sysVer = subprocess.Popen(sysVerCmd, shell=True, stdout=subprocess.PIPE).stdout.read() + if sys.version_info >= (3,): + sysVer = sysVer.decode(locale.getpreferredencoding(), 'replace') + return str(sysVer).rstrip() + class SetupTest(unittest.TestCase): def setUp(self): @@ -75,6 +86,12 @@ class SetupTest(unittest.TestCase): raise unittest.SkipTest( "Seems to be running not out of source distribution" " -- cannot locate setup.py") + # compare current version of python installed resp. active one: + sysVer = _getSysPythonVersion() + if sysVer != str(tuple(sys.version_info)): + raise unittest.SkipTest( + "Seems to be running with python distribution %s" + " -- install can be tested only with system distribution %s" % (str(tuple(sys.version_info)), sysVer)) def testSetupInstallRoot(self): if not self.setup: From 72f29e90619dbc02bc3192c9f44ddf38195a2393 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Jul 2015 20:23:07 +0200 Subject: [PATCH 135/143] asyncserver (asyncore) code fixed and test cases repaired (always delete temp files, wait for end of thread/server, etc) definitely closes gh-161, also other usage of asyncore event loop (in test_smtp.py) repair cache in ipToName (can returns None), precaching of invalid IPs (according to RFC 5737) to stop endless wait for resolving it in test cases. --- fail2ban/server/asyncserver.py | 67 ++++++++++++++++++++-------- fail2ban/server/filter.py | 4 +- fail2ban/tests/action_d/test_smtp.py | 10 ++--- fail2ban/tests/servertestcase.py | 6 +++ fail2ban/tests/sockettestcase.py | 32 +++++++++++-- fail2ban/tests/utils.py | 13 ++++++ 6 files changed, 103 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index a9be0ae2..ad37544a 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -27,12 +27,14 @@ __license__ = "GPL" from pickle import dumps, loads, HIGHEST_PROTOCOL import asynchat import asyncore +import errno import fcntl import os import socket import sys import traceback +from .utils import Utils from ..protocol import CSPROTO from ..helpers import getLogger,formatExceptionInfo @@ -89,6 +91,29 @@ class RequestHandler(asynchat.async_chat): self.close() +def loop(active, timeout=None, use_poll=False): + # Use poll instead of loop, because of recognition of active flag, + # because of loop timeout mistake: different in poll and poll2 (sec vs ms), + # and to prevent sporadical errors like EBADF 'Bad file descriptor' etc. (see gh-161) + if timeout is None: + timeout = Utils.DEFAULT_SLEEP_TIME + poll = asyncore.poll + if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover + logSys.debug('Server listener (select) uses poll') + # poll2 expected a timeout in milliseconds (but poll and loop in seconds): + timeout = float(timeout) / 1000 + poll = asyncore.poll2 + # Poll as long as active: + while active(): + try: + poll(timeout) + except Exception as e: # pragma: no cover + if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor') + logSys.info('Server connection was closed: %s', str(e)) + else: + logSys.error('Server connection was closed: %s', str(e)) + + ## # Asynchronous server class. # @@ -102,6 +127,7 @@ class AsyncServer(asyncore.dispatcher): self.__transmitter = transmitter self.__sock = "/var/run/fail2ban/fail2ban.sock" self.__init = False + self.__active = False ## # Returns False as we only read the socket first. @@ -129,7 +155,7 @@ class AsyncServer(asyncore.dispatcher): # @param sock: socket file. # @param force: remove the socket file if exists. - def start(self, sock, force): + def start(self, sock, force, use_poll=False): self.__sock = sock # Remove socket if os.path.exists(sock): @@ -149,28 +175,31 @@ class AsyncServer(asyncore.dispatcher): AsyncServer.__markCloseOnExec(self.socket) self.listen(1) # Sets the init flag. - self.__init = True - # TODO Add try..catch - # There's a bug report for Python 2.6/3.0 that use_poll=True yields some 2.5 incompatibilities: - if (sys.version_info >= (2, 7) and sys.version_info < (2, 8)) \ - or (sys.version_info >= (3, 4)): # if python 2.7 ... - logSys.debug("Detected Python 2.7. asyncore.loop() using poll") - asyncore.loop(use_poll=True) # workaround for the "Bad file descriptor" issue on Python 2.7, gh-161 - else: - asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 - + self.__init = self.__active = True + # Event loop as long as active: + loop(lambda: self.__active) + # Cleanup all + self.stop() + + + def close(self): + if self.__active: + asyncore.dispatcher.close(self) + # Remove socket (file) only if it was created: + if self.__init and os.path.exists(self.__sock): + logSys.debug("Removed socket file " + self.__sock) + os.remove(self.__sock) + logSys.debug("Socket shutdown") + self.__active = False + ## # Stops the communication server. def stop(self): - if self.__init: - # Only closes the socket if it was initialized first. - self.close() - # Remove socket - if os.path.exists(self.__sock): - logSys.debug("Removed socket file " + self.__sock) - os.remove(self.__sock) - logSys.debug("Socket shutdown") + self.close() + + def isActive(self): + return self.__active ## # Marks socket as close-on-exec to avoid leaking file descriptors when diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d1ac3680..8dbb6a7a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1023,8 +1023,8 @@ class DNSUtils: @staticmethod def ipToName(ip): # cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns: - v = DNSUtils.CACHE_ipToName.get(ip) - if v is not None: + v = DNSUtils.CACHE_ipToName.get(ip, ()) + if v != (): return v # retrieve name try: diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 27442832..37fe0138 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -19,7 +19,6 @@ import os import smtpd -import asyncore import threading import unittest import sys @@ -30,7 +29,7 @@ else: from ..dummyjail import DummyJail -from ..utils import CONFIG_DIR +from ..utils import CONFIG_DIR, asyncserver class TestSMTPServer(smtpd.SMTPServer): @@ -46,8 +45,6 @@ class SMTPActionTest(unittest.TestCase): def setUp(self): """Call before every test case.""" - unittest.F2B.SkipIfNoNetwork() - self.jail = DummyJail() pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py") pythonModuleName = os.path.basename(pythonModule.rstrip(".py")) @@ -64,13 +61,16 @@ class SMTPActionTest(unittest.TestCase): self.action = customActionModule.Action( self.jail, "test", host="127.0.0.1:%i" % port) + ## because of bug in loop (see loop in asyncserver.py) use it's loop instead of asyncore.loop: + self._active = True self._loop_thread = threading.Thread( - target=asyncore.loop, kwargs={'timeout': 1}) + target=asyncserver.loop, kwargs={'active': lambda: self._active}) self._loop_thread.start() def tearDown(self): """Call after every test case.""" self.smtpd.close() + self._active = False self._loop_thread.join() def testStart(self): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index d26d7562..87ca4289 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -62,11 +62,14 @@ class TransmitterBase(unittest.TestCase): def setUp(self): """Call before every test case.""" self.transm = self.server._Server__transm + self.tmp_files = [] sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter') os.close(sock_fd) + self.tmp_files.append(sock_name) pidfile_fd, pidfile_name = tempfile.mkstemp( 'fail2ban.pid', 'transmitter') os.close(pidfile_fd) + self.tmp_files.append(pidfile_name) self.server.start(sock_name, pidfile_name, force=False) self.jailName = "TestJail1" self.server.addJail(self.jailName, "auto") @@ -74,6 +77,9 @@ class TransmitterBase(unittest.TestCase): def tearDown(self): """Call after every test case.""" self.server.quit() + for f in self.tmp_files: + if os.path.exists(f): + os.remove(f) def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False): """Process set/get commands and compare both return values diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index a9408fde..5bf0be57 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -55,6 +55,24 @@ class Socket(unittest.TestCase): """Test transmitter proceed method which just returns first arg""" return message + def testStopPerCloseUnexpected(self): + # start in separate thread : + serverThread = threading.Thread( + target=self.server.start, args=(self.sock_name, False)) + serverThread.daemon = True + serverThread.start() + self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10))) + # unexpected stop directly after start: + self.server.close() + # wait for end of thread : + Utils.wait_for(lambda: not serverThread.isAlive() + or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10)) + self.assertFalse(serverThread.isAlive()) + # clean : + self.server.stop() + self.assertFalse(self.server.isActive()) + self.assertFalse(os.path.exists(self.sock_name)) + def _serverSocket(self): try: return CSocket(self.sock_name) @@ -66,6 +84,7 @@ class Socket(unittest.TestCase): target=self.server.start, args=(self.sock_name, False)) serverThread.daemon = True serverThread.start() + self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10))) time.sleep(Utils.DEFAULT_SLEEP_TIME) client = Utils.wait_for(self._serverSocket, 2) @@ -78,7 +97,11 @@ class Socket(unittest.TestCase): client.close() self.server.stop() - serverThread.join(Utils.DEFAULT_SLEEP_TIME) + # wait for end of thread : + Utils.wait_for(lambda: not serverThread.isAlive() + or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10)) + self.assertFalse(serverThread.isAlive()) + self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) def testSocketForce(self): @@ -92,10 +115,13 @@ class Socket(unittest.TestCase): target=self.server.start, args=(self.sock_name, True)) serverThread.daemon = True serverThread.start() - time.sleep(Utils.DEFAULT_SLEEP_TIME) + self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10))) self.server.stop() - serverThread.join(Utils.DEFAULT_SLEEP_TIME) + # wait for end of thread : + Utils.wait_for(lambda: not serverThread.isAlive() + or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10)) + self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 338ccec1..eb4e6e2e 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -35,6 +35,8 @@ from ..helpers import getLogger from ..server.filter import DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils +# for action_d.test_smtp : +from ..server import asyncserver logSys = getLogger(__name__) @@ -61,6 +63,10 @@ class F2B(optparse.Values): pass def SkipIfNoNetwork(self): pass + def maxWaitTime(self,wtime): + if self.fast: + wtime = float(wtime) / 10 + return wtime def initTests(opts): @@ -87,6 +93,13 @@ def initTests(opts): def F2B_SkipIfNoNetwork(): raise unittest.SkipTest('Skip test because of "--no-network"') unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork + # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): + c = DNSUtils.CACHE_ipToName + for i in xrange(255): + c.set('192.0.2.%s' % i, None) + c.set('198.51.100.%s' % i, None) + c.set('203.0.113.%s' % i, None) + def mtimesleep(): # no sleep now should be necessary since polling tracks now not only From f143ae479cee8510ddc7385cb84228a4823b58aa Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 1 Sep 2015 20:10:47 +0200 Subject: [PATCH 136/143] code and config review after rebase to current master --- fail2ban/server/server.py | 5 ++--- fail2ban/tests/filtertestcase.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 86197bc5..3bdfd71b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -326,9 +326,8 @@ class Server: def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 - for j in self.__jails: - j = self.__jails[j] - if not j.isAlive(): + for jail in self.__jails.values(): + if not jail.isAlive(): return 0 return 1 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 11642d2c..5edfd267 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1302,9 +1302,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res, []) def testIpToName(self): + unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertEqual(res, 'google-public-dns-b.google.com') - unittest.F2B.SkipIfNoNetwork() # invalid ip (TEST-NET-1 according to RFC 5737) res = DNSUtils.ipToName('192.0.2.0') self.assertEqual(res, None) From f7cc55103c77cd4dc0502268ea05215c32a502c6 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 17 Nov 2015 08:56:41 +0100 Subject: [PATCH 137/143] optimized FailManager: increase performance, try to prevent memory leakage (don't copy failures resp. it list on some operations) --- fail2ban/server/failmanager.py | 87 +++++++++++---------------- fail2ban/tests/failmanagertestcase.py | 4 +- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 4d006098..905553d4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -44,53 +44,30 @@ class FailManager: self.__failTotal = 0 def setFailTotal(self, value): - try: - self.__lock.acquire() + with self.__lock: self.__failTotal = value - finally: - self.__lock.release() def getFailTotal(self): - try: - self.__lock.acquire() + with self.__lock: return self.__failTotal - finally: - self.__lock.release() def setMaxRetry(self, value): - try: - self.__lock.acquire() - self.__maxRetry = value - finally: - self.__lock.release() + self.__maxRetry = value def getMaxRetry(self): - try: - self.__lock.acquire() - return self.__maxRetry - finally: - self.__lock.release() + return self.__maxRetry def setMaxTime(self, value): - try: - self.__lock.acquire() - self.__maxTime = value - finally: - self.__lock.release() + self.__maxTime = value def getMaxTime(self): - try: - self.__lock.acquire() - return self.__maxTime - finally: - self.__lock.release() + return self.__maxTime def addFailure(self, ticket, count=1): attempts = 1 - try: - self.__lock.acquire() + with self.__lock: ip = ticket.getIP() - if ip in self.__failList: + try: fData = self.__failList[ip] # if the same object: if fData is ticket: @@ -103,7 +80,7 @@ class FailManager: fData.setRetry(0) fData.inc(matches, 1, count) fData.setLastTime(unixTime) - else: + except KeyError: # if already FailTicket - add it direct, otherwise create (using copy all ticket data): if isinstance(ticket, FailTicket): fData = ticket; @@ -124,42 +101,46 @@ class FailManager: for k,v in self.__failList.iteritems()]) logSys.debug("Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) - finally: - self.__lock.release() return attempts def size(self): - try: - self.__lock.acquire() + with self.__lock: return len(self.__failList) - finally: - self.__lock.release() def cleanup(self, time): - try: - self.__lock.acquire() - tmp = self.__failList.copy() - for item in tmp: - if tmp[item].getLastTime() < time - self.__maxTime: - self.__delFailure(item) - finally: - self.__lock.release() + with self.__lock: + todelete = [ip for ip,item in self.__failList.iteritems() \ + if item.getLastTime() + self.__maxTime <= time] + if len(todelete) == len(self.__failList): + # remove all: + self.__failList = dict() + elif not len(todelete): + # nothing: + return + if len(todelete) / 2.0 <= len(self.__failList) / 3.0: + # few as 2/3 should be removed - remove particular items: + for ip in todelete: + del self.__failList[ip] + else: + # create new dictionary without items to be deleted: + self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \ + if item.getLastTime() + self.__maxTime > time) - def __delFailure(self, ip): - if ip in self.__failList: - del self.__failList[ip] + def delFailure(self, ip): + with self.__lock: + try: + del self.__failList[ip] + except KeyError: + pass def toBan(self, ip=None): - try: - self.__lock.acquire() + with self.__lock: for ip in ([ip] if ip != None and ip in self.__failList else self.__failList): data = self.__failList[ip] if data.getRetry() >= self.__maxRetry: del self.__failList[ip] return data raise FailManagerEmpty - finally: - self.__lock.release() class FailManagerEmpty(Exception): diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index a8c0f6fc..5a6e95e7 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -68,11 +68,11 @@ class AddFailure(unittest.TestCase): self.assertEqual(self.__failManager.getMaxTime(), 13) self.__failManager.setMaxTime(600) - def _testDel(self): + def testDel(self): self.__failManager.delFailure('193.168.0.128') self.__failManager.delFailure('111.111.1.111') - self.assertEqual(self.__failManager.size(), 1) + self.assertEqual(self.__failManager.size(), 2) def testCleanupOK(self): timestamp = 1167606999.0 From a10eb39bbef646c5d1790db16882973028f0f695 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 19 Nov 2015 04:26:35 +0100 Subject: [PATCH 138/143] test cases extended with memory leakage check --- fail2ban/server/failmanager.py | 5 +- fail2ban/tests/failmanagertestcase.py | 105 ++++++++++++++++++++++++++ fail2ban/tests/utils.py | 1 + 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 905553d4..07be0a4c 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -32,6 +32,7 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = logging.DEBUG class FailManager: @@ -93,13 +94,13 @@ class FailManager: attempts = fData.getRetry() self.__failTotal += 1 - if logSys.getEffectiveLevel() <= logging.DEBUG: + if logSys.getEffectiveLevel() <= logLevel: # yoh: Since composing this list might be somewhat time consuming # in case of having many active failures, it should be ran only # if debug level is "low" enough failures_summary = ', '.join(['%s:%d' % (k, v.getRetry()) for k,v in self.__failList.iteritems()]) - logSys.debug("Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" + logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) return attempts diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 5a6e95e7..2d4ce430 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -26,6 +26,7 @@ __license__ = "GPL" import unittest +from ..server import failmanager from ..server.failmanager import FailManager, FailManagerEmpty from ..server.ticket import FailTicket @@ -120,3 +121,107 @@ class AddFailure(unittest.TestCase): ticket = self.__failManager.toBan() self.assertNotEqual(ticket.getIP(), "100.100.10.10") self.assertRaises(FailManagerEmpty, self.__failManager.toBan) + + +class FailmanagerComplex(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + super(FailmanagerComplex, self).setUp() + self.__failManager = FailManager() + # down logging level for all this tests, because of extremely large failure count (several GB on heavydebug) + self.__saved_ll = failmanager.logLevel + failmanager.logLevel = 3 + + def tearDown(self): + super(FailmanagerComplex, self).tearDown() + # restore level + failmanager.logLevel = self.__saved_ll + + @staticmethod + def _ip_range(maxips): + class _ip(list): + def __str__(self): + return '.'.join(map(str, self)) + def __repr__(self): + return str(self) + def __key__(self): + return str(self) + def __hash__(self): + #return (int)(struct.unpack('I', struct.pack("BBBB",*self))[0]) + return (int)(self[0] << 24 | self[1] << 16 | self[2] << 8 | self[3]) + i = 0 + c = [127,0,0,0] + while i < maxips: + for n in range(3,0,-1): + if c[n] < 255: + c[n] += 1 + break + c[n] = 0 + yield (i, _ip(c)) + i += 1 + + def testCheckIPGenerator(self): + for i, ip in self._ip_range(65536 if not unittest.F2B.fast else 1000): + if i == 254: + self.assertEqual(str(ip), '127.0.0.255') + elif i == 255: + self.assertEqual(str(ip), '127.0.1.0') + elif i == 1000: + self.assertEqual(str(ip), '127.0.3.233') + elif i == 65534: + self.assertEqual(str(ip), '127.0.255.255') + elif i == 65535: + self.assertEqual(str(ip), '127.1.0.0') + + def testFailuresMemLeak1(self): + # use factor (divisor) instead unittest.F2B.SkipIfFast() : + modeDiv = 1 + if unittest.F2B.fast: # pragma: no cover + modeDiv = 10 + self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv) + import gc + gc.collect() + timestamp = 1167606999.0 + ticktime = timestamp-(1000 // modeDiv) + for i, ip in self._ip_range(30000 // modeDiv): + t = FailTicket(ip, ticktime, list({'match': i})) + ticktime += 1 + self.__failManager.addFailure(t) + if i % (500 // modeDiv) == 0: + self.__failManager.cleanup(timestamp) + if i % (888 // modeDiv) == 0: + timestamp += (1000 // modeDiv) + ticktime = timestamp-(1000 // modeDiv) + self.assertFalse(gc.collect()) + self.__failManager.cleanup(timestamp) + ticktime -= timestamp - self.__failManager.getMaxTime() + 1 + self.assertEqual(self.__failManager.size(), ticktime if ticktime > 0 else 0) + self.assertFalse(gc.collect()) + + def testFailuresMemLeak2(self): + # use factor (divisor) instead unittest.F2B.SkipIfFast() : + modeDiv = 1 + if unittest.F2B.fast: # pragma: no cover + modeDiv = 10 + self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv) + import gc + gc.collect() + timestamp = 1167606999.0 + ticktime = timestamp-(1000 // modeDiv) + for i, ip in self._ip_range(10000 // modeDiv): + for j in range(0, 5): + t = FailTicket(ip, ticktime, list({'match': i})) + ticktime += 1 + self.__failManager.addFailure(t) + if i % (500 // modeDiv) == 0: + self.__failManager.cleanup(timestamp) + if i % (888 // modeDiv) == 0 or i % (1500 // modeDiv) == 0: + timestamp += (1000 // modeDiv) + ticktime = timestamp-(1000 // modeDiv) + self.assertFalse(gc.collect()) + timestamp += 20000 + self.__failManager.cleanup(timestamp) + self.assertEqual(self.__failManager.size(), 0) + self.assertFalse(gc.collect()) + diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index eb4e6e2e..c33c2b19 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -176,6 +176,7 @@ def gatherTests(regexps=None, opts=None): tests.addTest(unittest.makeSuite(tickettestcase.TicketTests)) # FailManager tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(failmanagertestcase.FailmanagerComplex)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) try: From 6406f6f5600deaf1d19f793cef8e30a1f55bf43f Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2015 12:48:07 +0100 Subject: [PATCH 139/143] background servicing (temporally executed from failmanager): prevents memory leak on some platforms/python versions, using forced GC in periodic intervals (latency and threshold); Side effect: GC is disabled now inside fail2ban-server (to avoid multiple garbage collect) --- fail2ban/helpers.py | 58 +++++++++++++++++++++++++-- fail2ban/server/failmanager.py | 9 ++++- fail2ban/tests/failmanagertestcase.py | 23 +++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 8e1b0e32..b5ffe601 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -20,11 +20,16 @@ __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko" __license__ = "GPL" -import sys -import os -import traceback -import re +import gc import logging +import os +import re +import sys +import traceback + +from threading import Lock + +from .server.mytime import MyTime def formatExceptionInfo(): @@ -137,3 +142,48 @@ def splitcommaspace(s): if not s: return [] return filter(bool, re.split('[ ,]', s)) + + +class BgService(object): + """Background servicing + + Prevents memory leak on some platforms/python versions, + using forced GC in periodical intervals. + """ + + _mutex = Lock() + _instance = None + def __new__(cls): + if not cls._instance: + cls._instance = \ + super(BgService, cls).__new__(cls) + return cls._instance + + def __init__(self): + self.__serviceTime = -0x7fffffff + self.__periodTime = 30 + self.__threshold = 100; + self.__count = self.__threshold; + if hasattr(gc, 'set_threshold'): + gc.set_threshold(0) + gc.disable() + + def service(self, force=False, wait=False): + self.__count -= 1 + # avoid locking if next service time don't reached + if not force and (self.__count > 0 or MyTime.time() < self.__serviceTime): + return False + # return immediately if mutex already locked (other thread in servicing): + if not BgService._mutex.acquire(wait): + return False + try: + # check again in lock: + if MyTime.time() < self.__serviceTime: + return False + gc.collect() + self.__serviceTime = MyTime.time() + self.__periodTime + self.__count = self.__threshold + return True + finally: + BgService._mutex.release() + return False diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 07be0a4c..45f3a393 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -28,7 +28,7 @@ from threading import Lock import logging from .ticket import FailTicket -from ..helpers import getLogger +from ..helpers import getLogger, BgService # Gets the instance of the logger. logSys = getLogger(__name__) @@ -43,6 +43,7 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 + self.__bgSvc = BgService() def setFailTotal(self, value): with self.__lock: @@ -102,6 +103,8 @@ class FailManager: for k,v in self.__failList.iteritems()]) logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) + + self.__bgSvc.service() return attempts def size(self): @@ -126,6 +129,7 @@ class FailManager: # create new dictionary without items to be deleted: self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \ if item.getLastTime() + self.__maxTime > time) + self.__bgSvc.service() def delFailure(self, ip): with self.__lock: @@ -141,7 +145,8 @@ class FailManager: if data.getRetry() >= self.__maxRetry: del self.__failList[ip] return data - raise FailManagerEmpty + self.__bgSvc.service() + raise FailManagerEmpty class FailManagerEmpty(Exception): diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 2d4ce430..78f7b509 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -122,6 +122,29 @@ class AddFailure(unittest.TestCase): self.assertNotEqual(ticket.getIP(), "100.100.10.10") self.assertRaises(FailManagerEmpty, self.__failManager.toBan) + def testBgService(self): + bgSvc = self.__failManager._FailManager__bgSvc + failManager2nd = FailManager() + # test singleton (same object): + bgSvc2 = failManager2nd._FailManager__bgSvc + self.assertTrue(id(bgSvc) == id(bgSvc2)) + bgSvc2 = None + # test service : + self.assertTrue(bgSvc.service(True, True)) + self.assertFalse(bgSvc.service()) + # bypass threshold and time: + for i in range(1, bgSvc._BgService__threshold): + self.assertFalse(bgSvc.service()) + # bypass time check: + bgSvc._BgService__serviceTime = -0x7fffffff + self.assertTrue(bgSvc.service()) + # bypass threshold and time: + bgSvc._BgService__serviceTime = -0x7fffffff + for i in range(1, bgSvc._BgService__threshold): + self.assertFalse(bgSvc.service()) + self.assertTrue(bgSvc.service(False, True)) + self.assertFalse(bgSvc.service(False, True)) + class FailmanagerComplex(unittest.TestCase): From 935d79eaaedef532230ab72a8256be713b27de1f Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 29 Nov 2015 01:04:55 +0100 Subject: [PATCH 140/143] 1) prevents a bug by logging stdout/stderr if retcode still None: ``` in executeCmd if retcode < 0: TypeError: unorderable types: NoneType() < int() ``` 2) prevents a rarely test case bug of testExecuteTimeoutWithNastyChildren, because no stdout (Resource temporarily unavailable), possible no flush by IO of the killing process; --- fail2ban/server/utils.py | 4 ++-- fail2ban/tests/actiontestcase.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 262b303d..45d1c09d 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -159,7 +159,7 @@ class Utils(): # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode. if popen.stdout: try: - if retcode < 0: + if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stdout, False) stdout = popen.stdout.read() except IOError as e: @@ -169,7 +169,7 @@ class Utils(): popen.stdout.close() if popen.stderr: try: - if retcode < 0: + if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stderr, False) stderr = popen.stderr.read() except IOError as e: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 8c9b5ef1..6d8fcc82 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -247,7 +247,7 @@ class CommandActionTest(LogCaptureTestCase): cpid = getnastypid() # Verify that the process itself got killed self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3)) # process should have been killed - self.assertLogged('my pid ') + self.assertLogged('my pid ', 'Resource temporarily unavailable') self.assertLogged('timed out') self.assertLogged('killed with SIGTERM', 'killed with SIGKILL') @@ -261,7 +261,7 @@ class CommandActionTest(LogCaptureTestCase): cpid = getnastypid() # Verify that the process itself got killed self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3)) - self.assertLogged('my pid ') + self.assertLogged('my pid ', 'Resource temporarily unavailable') self.assertLogged('timed out') self.assertLogged('killed with SIGTERM', 'killed with SIGKILL') From e065941ac52c2566595d9dfa29f07e5fa419cf1c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Dec 2015 21:35:37 +0100 Subject: [PATCH 141/143] use "maxEntries" (currently 50 as default) as range for max number of the last matches/failures, fail2ban will hold per IP in the list of failures in failmanager resp. in the database; prevents out of memory situation if many IP's makes extremely many failures (or very large files since last fail2ban run); closes gh-1277 todo: parameter `maxentries` should be configurable (jail.conf resp. fail2ban.conf); todo: adjust ban-time-incr branch by merge (table "bips"). --- fail2ban/server/database.py | 12 +++++- fail2ban/server/failmanager.py | 16 +++++-- fail2ban/tests/databasetestcase.py | 31 ++++++++++++++ fail2ban/tests/failmanagertestcase.py | 62 ++++++++++++++++++++++++--- 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 3b419ed3..b7fd4d47 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -163,6 +163,7 @@ class Fail2BanDb(object): def __init__(self, filename, purgeAge=24*60*60): + self.maxEntries = 50 try: self._lock = RLock() self._db = sqlite3.connect( @@ -454,7 +455,7 @@ class Fail2BanDb(object): if ip is not None: query += " AND ip=?" queryArgs.append(ip) - query += " ORDER BY ip, timeofban" + query += " ORDER BY ip, timeofban desc" return cur.execute(query, queryArgs) @@ -535,7 +536,14 @@ class Fail2BanDb(object): matches = [] failures = 0 tickdata = {} - matches.extend(data.get('matches', ())) + m = data.get('matches', []) + # pre-insert "maxadd" enries (because tickets are ordered desc by time) + maxadd = self.maxEntries - len(matches) + if maxadd > 0: + if len(m) <= maxadd: + matches = m + matches + else: + matches = m[-maxadd:] + matches failures += data.get('failures', 1) tickdata.update(data.get('data', {})) prev_timeofban = timeofban diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 45f3a393..ae97b36a 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -43,6 +43,7 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 + self.maxEntries = 50 self.__bgSvc = BgService() def setFailTotal(self, value): @@ -71,17 +72,26 @@ class FailManager: ip = ticket.getIP() try: fData = self.__failList[ip] - # if the same object: + # if the same object - the same matches but +1 attempt: if fData is ticket: matches = None + attempt = 1 else: + # will be incremented / extended (be sure we have at least +1 attempt): matches = ticket.getMatches() + attempt = ticket.getAttempt() + if attempt <= 0: + attempt += 1 unixTime = ticket.getTime() + fData.setLastTime(unixTime) if fData.getLastReset() < unixTime - self.__maxTime: fData.setLastReset(unixTime) fData.setRetry(0) - fData.inc(matches, 1, count) - fData.setLastTime(unixTime) + fData.inc(matches, attempt, count) + # truncate to maxEntries: + matches = fData.getMatches() + if len(matches) > self.maxEntries: + fData.setMatches(matches[-self.maxEntries:]) except KeyError: # if already FailTicket - add it direct, otherwise create (using copy all ticket data): if isinstance(ticket, FailTicket): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 6b85af56..5d83710f 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -251,6 +251,37 @@ class DatabaseTest(LogCaptureTestCase): # be returned self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=-1)), 2) + def testGetBansMerged_MaxEntries(self): + if Fail2BanDb is None: # pragma: no cover + return + self.testAddJail() + maxEntries = 2 + failures = ["abc\n", "123\n", "ABC\n", "1234\n"] + # add failures sequential: + i = 80 + for f in failures: + i -= 10 + ticket = FailTicket("127.0.0.1", MyTime.time() - i, [f]) + ticket.setAttempt(1) + self.db.addBan(self.jail, ticket) + # should retrieve 2 matches only, but count of all attempts: + self.db.maxEntries = maxEntries; + ticket = self.db.getBansMerged("127.0.0.1") + self.assertEqual(ticket.getIP(), "127.0.0.1") + self.assertEqual(ticket.getAttempt(), len(failures)) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + # add more failures at once: + ticket = FailTicket("127.0.0.1", MyTime.time() - 10, failures) + ticket.setAttempt(len(failures)) + self.db.addBan(self.jail, ticket) + # should retrieve 2 matches only, but count of all attempts: + self.db.maxEntries = maxEntries; + ticket = self.db.getBansMerged("127.0.0.1") + self.assertEqual(ticket.getAttempt(), 2 * len(failures)) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + def testGetBansMerged(self): if Fail2BanDb is None: # pragma: no cover return diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 78f7b509..3cac0540 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -35,6 +35,13 @@ class AddFailure(unittest.TestCase): def setUp(self): """Call before every test case.""" + self.__items = None + self.__failManager = FailManager() + + def tearDown(self): + """Call after every test case.""" + + def _addDefItems(self): self.__items = [[u'193.168.0.128', 1167605999.0], [u'193.168.0.128', 1167605999.0], [u'193.168.0.128', 1167605999.0], @@ -48,44 +55,87 @@ class AddFailure(unittest.TestCase): ['100.100.10.10', 1000001000.0], ['100.100.10.10', 1000001500.0], ['100.100.10.10', 1000002000.0]] - - self.__failManager = FailManager() for i in self.__items: self.__failManager.addFailure(FailTicket(i[0], i[1])) - def tearDown(self): - """Call after every test case.""" - def testFailManagerAdd(self): + self._addDefItems() self.assertEqual(self.__failManager.size(), 3) self.assertEqual(self.__failManager.getFailTotal(), 13) self.__failManager.setFailTotal(0) self.assertEqual(self.__failManager.getFailTotal(), 0) self.__failManager.setFailTotal(13) + def testFailManagerAdd_MaxEntries(self): + maxEntries = 2 + self.__failManager.maxEntries = maxEntries + failures = ["abc\n", "123\n", "ABC\n", "1234\n"] + # add failures sequential: + i = 80 + for f in failures: + i -= 10 + ticket = FailTicket("127.0.0.1", 1000002000 - i, [f]) + ticket.setAttempt(1) + self.__failManager.addFailure(ticket) + # + manFailList = self.__failManager._FailManager__failList + self.assertEqual(len(manFailList), 1) + ticket = manFailList["127.0.0.1"] + # should retrieve 2 matches only, but count of all attempts (4): + self.assertEqual(ticket.getAttempt(), len(failures)) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + # add more failures at once: + ticket = FailTicket("127.0.0.1", 1000002000 - 10, failures) + ticket.setAttempt(len(failures)) + self.__failManager.addFailure(ticket) + # + manFailList = self.__failManager._FailManager__failList + self.assertEqual(len(manFailList), 1) + ticket = manFailList["127.0.0.1"] + # should retrieve 2 matches only, but count of all attempts (8): + self.assertEqual(ticket.getAttempt(), 2 * len(failures)) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + # add self ticket again: + self.__failManager.addFailure(ticket) + # + manFailList = self.__failManager._FailManager__failList + self.assertEqual(len(manFailList), 1) + ticket = manFailList["127.0.0.1"] + # same matches, but +1 attempt (9) + self.assertEqual(ticket.getAttempt(), 2 * len(failures) + 1) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + def testFailManagerMaxTime(self): + self._addDefItems() self.assertEqual(self.__failManager.getMaxTime(), 600) self.__failManager.setMaxTime(13) self.assertEqual(self.__failManager.getMaxTime(), 13) self.__failManager.setMaxTime(600) def testDel(self): + self._addDefItems() self.__failManager.delFailure('193.168.0.128') self.__failManager.delFailure('111.111.1.111') self.assertEqual(self.__failManager.size(), 2) def testCleanupOK(self): + self._addDefItems() timestamp = 1167606999.0 self.__failManager.cleanup(timestamp) self.assertEqual(self.__failManager.size(), 0) def testCleanupNOK(self): + self._addDefItems() timestamp = 1167605990.0 self.__failManager.cleanup(timestamp) self.assertEqual(self.__failManager.size(), 2) def testbanOK(self): + self._addDefItems() self.__failManager.setMaxRetry(5) #ticket = FailTicket('193.168.0.128', None) ticket = self.__failManager.toBan() @@ -112,10 +162,12 @@ class AddFailure(unittest.TestCase): 'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5 matches=[]') def testbanNOK(self): + self._addDefItems() self.__failManager.setMaxRetry(10) self.assertRaises(FailManagerEmpty, self.__failManager.toBan) def testWindow(self): + self._addDefItems() ticket = self.__failManager.toBan() self.assertNotEqual(ticket.getIP(), "100.100.10.10") ticket = self.__failManager.toBan() From 3f2b58e9730e0c743fd0db2eeacd34585b7fa36b Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 23 Dec 2015 18:49:30 +0100 Subject: [PATCH 142/143] datedetector: matchTime, template list etc. rewritten because of performance degradation (without sorting templates now) - in-place reordering using hits and last used time; todo: rewrite "reGroupDictStrptime" because may be too slow; --- fail2ban/client/fail2banregex.py | 4 +- fail2ban/server/datedetector.py | 65 ++++++++++++++++---------- fail2ban/server/datetemplate.py | 1 + fail2ban/server/filtergamin.py | 1 - fail2ban/server/filterpoll.py | 1 - fail2ban/server/filterpyinotify.py | 1 - fail2ban/tests/datedetectortestcase.py | 9 ---- fail2ban/tests/filtertestcase.py | 35 +++++++++++++- 8 files changed, 75 insertions(+), 42 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index d0cdab84..1fc5316e 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -366,7 +366,7 @@ class Fail2banRegex(object): def process(self, test_lines): t0 = time.time() - for line_no, line in enumerate(test_lines): + for line in test_lines: if isinstance(line, tuple): line_datetimestripped, ret = self.testRegex( line[0], line[1]) @@ -398,8 +398,6 @@ class Fail2banRegex(object): self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 - if line_no % 10 == 0 and self._filter.dateDetector is not None: - self._filter.dateDetector.sortTemplate() self._time_elapsed = time.time() - t0 def printLines(self, ltype): diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 4bdcefd2..4004d5c5 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -21,6 +21,8 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import time + from threading import Lock from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch @@ -44,6 +46,8 @@ class DateDetector(object): self.__lock = Lock() self.__templates = list() self.__known_names = set() + # time the template was long unused (currently 300 == 5m): + self.__unusedTime = 300 def _appendTemplate(self, template): name = template.name @@ -153,22 +157,29 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ - self.__lock.acquire() - try: + i = 0 + with self.__lock: for template in self.__templates: match = template.matchDate(line) if not match is None: if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, "Matched time template %s", template.name) template.hits += 1 + template.lastUsed = time.time() + # if not first - try to reorder current template (bubble up), they will be not sorted anymore: + if i: + self._reorderTemplate(i) + # return tuple with match and template reference used for parsing: return (match, template) - return (None, None) - finally: - self.__lock.release() + i += 1 + # not found: + return (None, None) def getTime(self, line): """Attempts to return the date on a log line using templates. + Obsolete: Use "getTime2" instead. + This uses the templates' `getDate` method in an attempt to find a date. @@ -183,8 +194,7 @@ class DateDetector(object): The Unix timestamp returned from the first successfully matched template or None if not found. """ - self.__lock.acquire() - try: + with self.__lock: for template in self.__templates: try: date = template.getDate(line) @@ -197,8 +207,6 @@ class DateDetector(object): except ValueError: # pragma: no cover pass return None - finally: - self.__lock.release() def getTime2(self, line, timeMatch = None): """Attempts to return the date on a log line using given template. @@ -232,21 +240,28 @@ class DateDetector(object): return date return self.getTime(line) - def sortTemplate(self): - """Sort the date templates by number of hits + def _reorderTemplate(self, num): + """Reorder template (bubble up) in template list if hits grows enough. - Sort the template lists using the hits score. This method is not - called in this object and thus should be called from time to time. - This ensures the most commonly matched templates are checked first, - improving performance of matchTime and getTime. + Parameters + ---------- + num : int + Index of template should be moved. """ - self.__lock.acquire() - try: - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Sorting the template list") - self.__templates.sort(key=lambda x: x.hits, reverse=True) - t = self.__templates[0] - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Winning template: %s with %d hits", t.name, t.hits) - finally: - self.__lock.release() + if num: + templates = self.__templates + template = templates[num] + ## current hits and time the template was long unused: + untime = template.lastUsed - self.__unusedTime + hits = template.hits + ## don't move too often (multiline logs resp. log's with different date patterns), + ## if template not used too long, replace it also : + if hits > templates[num-1].hits + 5 or templates[num-1].lastUsed < untime: + ## try to move faster (half of part to current template): + pos = num // 2 + ## if not larger - move slow (exact 1 position): + if hits <= templates[pos].hits or templates[pos].lastUsed < untime: + pos = num-1 + templates[pos], templates[num] = template, templates[pos] + + diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e1f6b3de..88e2ae3f 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -50,6 +50,7 @@ class DateTemplate(object): self._regex = "" self._cRegex = None self.hits = 0 + self.lastUsed = 0 @property def name(self): diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index b094cfeb..6b562a5c 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -84,7 +84,6 @@ class FilterGamin(FileFilter): self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) - self.dateDetector.sortTemplate() self.__modified = False ## diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index d7ff9075..c7b04970 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -118,7 +118,6 @@ class FilterPoll(FileFilter): self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) - self.dateDetector.sortTemplate() self.__modified = False logSys.debug( (self.jail is not None and self.jail.name or "jailless") + diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 100ad233..7bd638d9 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -108,7 +108,6 @@ class FilterPyinotify(FileFilter): self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) - self.dateDetector.sortTemplate() self.__modified = False def _addFileWatcher(self, path): diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 9a7d3e78..ab7310ba 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -143,13 +143,6 @@ class DateDetectorTest(LogCaptureTestCase): else: self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) - def testStableSortTemplate(self): - old_names = [x.name for x in self.__datedetector.templates] - self.__datedetector.sortTemplate() - # If there were no hits -- sorting should not change the order - for old_name, n in zip(old_names, self.__datedetector.templates): - self.assertEqual(old_name, n.name) # "Sort must be stable" - def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.__datedetector.appendTemplate, self.__datedetector.templates[0]) @@ -164,13 +157,11 @@ class DateDetectorTest(LogCaptureTestCase): ( logTime, logMatch ) = logdate self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') - self.__datedetector.sortTemplate() # confuse it with year being at the end for i in xrange(10): ( logTime, logMatch ) = self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(), '11/10/2012 02:37:17') - self.__datedetector.sortTemplate() # and now back to the original ( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 5edfd267..ae22b88f 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -87,8 +87,31 @@ def _maxWaitTime(wtime): return wtime -def _tm(time): - return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") +class _tmSerial(): + _last_s = -0x7fffffff + _last_m = -0x7fffffff + _str_s = "" + _str_m = "" + @staticmethod + def _tm(time): + # ## strftime it too slow for large time serializer : + # return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") + c = _tmSerial + sec = (time % 60) + if c._last_s == time - sec: + return "%s%02u" % (c._str_s, sec) + mt = (time % 3600) + if c._last_m == time - mt: + c._last_s = time - sec + c._str_s = "%s%02u:" % (c._str_m, mt // 60) + return "%s%02u" % (c._str_s, sec) + c._last_m = time - mt + c._str_m = datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:") + c._last_s = time - sec + c._str_s = "%s%02u:" % (c._str_m, mt // 60) + return "%s%02u" % (c._str_s, sec) + +_tm = _tmSerial._tm def _assert_equal_entries(utest, found, output, count=None): @@ -245,6 +268,14 @@ class BasicFilter(unittest.TestCase): 1) ) + def testTest_tm(self): + unittest.F2B.SkipIfFast() + ## test function "_tm" works correct (returns the same as slow strftime): + for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600): + tm = datetime.datetime.fromtimestamp(i).strftime("%Y-%m-%d %H:%M:%S") + if _tm(i) != tm: + self.assertEqual((_tm(i), i), (tm, i)) + class IgnoreIP(LogCaptureTestCase): From da51fbf9c626a206c67e6b9e7dec61eadec2f8ba Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Dec 2015 12:40:26 +0100 Subject: [PATCH 143/143] code review and performance optimization: - default date templates cache + regexp compiling on the fly (if required); - better usage of shared config over all test cases; - obsolete test cases removed (simple test of memleak/gc); - skip some slow test cases in fast mode (setup test, etc.); --- fail2ban/server/datedetector.py | 133 +++++++++++++++---------- fail2ban/server/datetemplate.py | 12 +-- fail2ban/tests/clientreadertestcase.py | 44 ++++---- fail2ban/tests/failmanagertestcase.py | 51 ---------- fail2ban/tests/misctestcase.py | 1 + fail2ban/tests/samplestestcase.py | 3 +- fail2ban/tests/servertestcase.py | 11 +- fail2ban/tests/utils.py | 1 + 8 files changed, 119 insertions(+), 137 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 4004d5c5..ae6d71d9 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -34,6 +34,82 @@ logSys = getLogger(__name__) logLevel = 6 +class DateDetectorCache(object): + def __init__(self): + self.__lock = Lock() + self.__templates = list() + + @property + def templates(self): + """List of template instances managed by the detector. + """ + with self.__lock: + if self.__templates: + return self.__templates + self._addDefaultTemplate() + return self.__templates + + def _cacheTemplate(self, template): + """Cache Fail2Ban's default template. + """ + if isinstance(template, str): + template = DatePatternRegex(template) + self.__templates.append(template) + + def _addDefaultTemplate(self): + """Add resp. cache Fail2Ban's default set of date templates. + """ + # asctime with optional day, subsecond and/or year: + # Sun Jan 23 21:59:59.011 2005 + self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # asctime with optional day, subsecond and/or year coming after day + # http://bugs.debian.org/798923 + # Sun Jan 23 2005 21:59:59.011 + self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") + # simple date, optional subsecond (proftpd): + # 2005-01-23 21:59:59 + # simple date: 2005/01/23 21:59:59 + # custom for syslog-ng 2006.12.21 06:43:20 + self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?") + # simple date too (from x11vnc): 23/01/2005 21:59:59 + # and with optional year given by 2 digits: 23/01/05 21:59:59 + # (See http://bugs.debian.org/537610) + # 17-07-2008 17:23:25 + self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") + # Apache format optional time zone: + # [31/Oct/2006:09:22:55 -0000] + # 26-Jul-2007 15:20:52 + self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") + # CPanel 05/20/2008:01:57:39 + self._cacheTemplate("%m/%d/%Y:%H:%M:%S") + # named 26-Jul-2007 15:20:52.252 + # roundcube 26-Jul-2007 15:20:52 +0200 + # 01-27-2012 16:22:44.252 + # subseconds explicit to avoid possible %m<->%d confusion + # with previous + self._cacheTemplate("%m-%d-%Y %H:%M:%S\.%f") + # TAI64N + template = DateTai64n() + template.name = "TAI64N" + self._cacheTemplate(template) + # Epoch + template = DateEpoch() + template.name = "Epoch" + self._cacheTemplate(template) + # ISO 8601 + self._cacheTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") + # Only time information in the log + self._cacheTemplate("^%H:%M:%S") + # <09/16/08@05:03:30> + self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>") + # MySQL: 130322 11:46:11 + self._cacheTemplate("^%y%m%d ?%H:%M:%S") + # Apache Tomcat + self._cacheTemplate("%b %d, %Y %I:%M:%S %p") + # ASSP: Apr-27-13 02:33:06 + self._cacheTemplate("^%b-%d-%y %H:%M:%S") + + class DateDetector(object): """Manages one or more date templates to find a date within a log line. @@ -41,6 +117,7 @@ class DateDetector(object): ---------- templates """ + _defCache = DateDetectorCache() def __init__(self): self.__lock = Lock() @@ -79,59 +156,9 @@ class DateDetector(object): def addDefaultTemplate(self): """Add Fail2Ban's default set of date templates. """ - self.__lock.acquire() - try: - # asctime with optional day, subsecond and/or year: - # Sun Jan 23 21:59:59.011 2005 - self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") - # asctime with optional day, subsecond and/or year coming after day - # http://bugs.debian.org/798923 - # Sun Jan 23 2005 21:59:59.011 - self.appendTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") - # simple date, optional subsecond (proftpd): - # 2005-01-23 21:59:59 - # simple date: 2005/01/23 21:59:59 - # custom for syslog-ng 2006.12.21 06:43:20 - self.appendTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?") - # simple date too (from x11vnc): 23/01/2005 21:59:59 - # and with optional year given by 2 digits: 23/01/05 21:59:59 - # (See http://bugs.debian.org/537610) - # 17-07-2008 17:23:25 - self.appendTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") - # Apache format optional time zone: - # [31/Oct/2006:09:22:55 -0000] - # 26-Jul-2007 15:20:52 - self.appendTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") - # CPanel 05/20/2008:01:57:39 - self.appendTemplate("%m/%d/%Y:%H:%M:%S") - # named 26-Jul-2007 15:20:52.252 - # roundcube 26-Jul-2007 15:20:52 +0200 - # 01-27-2012 16:22:44.252 - # subseconds explicit to avoid possible %m<->%d confusion - # with previous - self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f") - # TAI64N - template = DateTai64n() - template.name = "TAI64N" - self.appendTemplate(template) - # Epoch - template = DateEpoch() - template.name = "Epoch" - self.appendTemplate(template) - # ISO 8601 - self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") - # Only time information in the log - self.appendTemplate("^%H:%M:%S") - # <09/16/08@05:03:30> - self.appendTemplate("^<%m/%d/%y@%H:%M:%S>") - # MySQL: 130322 11:46:11 - self.appendTemplate("^%y%m%d ?%H:%M:%S") - # Apache Tomcat - self.appendTemplate("%b %d, %Y %I:%M:%S %p") - # ASSP: Apr-27-13 02:33:06 - self.appendTemplate("^%b-%d-%y %H:%M:%S") - finally: - self.__lock.release() + with self.__lock: + for template in DateDetector._defCache.templates: + self._appendTemplate(template) @property def templates(self): diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 88e2ae3f..746380ed 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -86,7 +86,6 @@ class DateTemplate(object): if (wordBegin and not re.search(r'^\^', regex)): regex = r'\b' + regex self._regex = regex - self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) regex = property(getRegex, setRegex, doc= """Regex used to search for date. @@ -95,6 +94,8 @@ class DateTemplate(object): def matchDate(self, line): """Check if regex for date matches on a log line. """ + if not self._cRegex: + self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE) dateMatch = self._cRegex.search(line) return dateMatch @@ -171,7 +172,7 @@ class DatePatternRegex(DateTemplate): regex pattern """ - _patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys()) + _patternRE = re.compile(r"%%(%%|[%s])" % "".join(timeRE.keys())) _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", 'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month", @@ -202,10 +203,9 @@ class DatePatternRegex(DateTemplate): @pattern.setter def pattern(self, pattern): self._pattern = pattern - self._name = re.sub( - self._patternRE, r'%(\1)s', pattern) % self._patternName - super(DatePatternRegex, self).setRegex( - re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE) + fmt = self._patternRE.sub(r'%(\1)s', pattern) + self._name = fmt % self._patternName + super(DatePatternRegex, self).setRegex(fmt % timeRE) def setRegex(self, value): raise NotImplementedError("Regex derived from pattern") diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d0c37a3e..78c9a582 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -38,12 +38,15 @@ from ..client.configurator import Configurator from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") +TEST_FILES_DIR_SHARE_CFG = {} from .utils import CONFIG_DIR +CONFIG_DIR_SHARE_CFG = unittest.F2B.share_config STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') +IMPERFECT_CONFIG_SHARE_CFG = {} class ConfigReaderTest(unittest.TestCase): @@ -162,14 +165,13 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} def testIncorrectJail(self): - jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg) + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) def testJailActionEmpty(self): - jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) + jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -177,7 +179,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): - jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) + jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -186,7 +188,7 @@ class JailReaderTest(LogCaptureTestCase): def testJailActionBrokenDef(self): jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, - share_config=self.__share_cfg) + share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -200,7 +202,7 @@ class JailReaderTest(LogCaptureTestCase): if STOCK: def testStockSSHJail(self): - jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -316,7 +318,7 @@ class FilterReaderTest(unittest.TestCase): self.assertEqual(sorted(filterReader.convert()), sorted(output)) filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"}, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) @@ -326,7 +328,7 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {}, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -335,7 +337,7 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -346,7 +348,7 @@ class FilterReaderTest(unittest.TestCase): filterName, filterOpt = JailReader.extractOptions( 'substition[honeypot=",", sweet="test,,test2"]') filterReader = FilterReader('substition', "jailname", filterOpt, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -355,13 +357,13 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionFail(self): # directly subst the same var : filterReader = FilterReader('substition', "jailname", {'honeypot': ''}, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) # cross subst the same var : filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}, - share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) @@ -404,6 +406,7 @@ class JailsReaderTestCache(LogCaptureTestCase): return cnt def testTestJailConfCache(self): + unittest.F2B.SkipIfFast() saved_ll = configparserinc.logLevel configparserinc.logLevel = logging.DEBUG basedir = tempfile.mkdtemp("fail2ban_conf") @@ -446,7 +449,6 @@ class JailsReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailsReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} def testProvidingBadBasedir(self): if not os.path.exists('/XXX'): @@ -454,7 +456,7 @@ class JailsReaderTest(LogCaptureTestCase): self.assertRaises(ValueError, reader.read) def testReadTestJailConf(self): - jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) + jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jails.read()) self.assertFalse(jails.getOptions()) self.assertRaises(ValueError, jails.convert) @@ -504,7 +506,7 @@ class JailsReaderTest(LogCaptureTestCase): msg="Action file %r is lacking [Init] section" % actionConfig) def testReadStockJailConf(self): - jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert() @@ -535,7 +537,7 @@ class JailsReaderTest(LogCaptureTestCase): # moreover we must have a file for it # and it must be readable as a Filter filterReader = FilterReader(filterName, jail, filterOpt, - share_config=self.__share_cfg, basedir=CONFIG_DIR) + share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR) self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName) # opens fine filterReader.getOptions({}) # reads fine @@ -554,8 +556,8 @@ class JailsReaderTest(LogCaptureTestCase): if actName == 'iptables-multiport': self.assertTrue('port' in actOpt) - actionReader = ActionReader( - actName, jail, {}, basedir=CONFIG_DIR) + actionReader = ActionReader(actName, jail, {}, + share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR) self.assertTrue(actionReader.read()) actionReader.getOptions({}) # populate _opts cmds = actionReader.convert() @@ -566,7 +568,7 @@ class JailsReaderTest(LogCaptureTestCase): # Verify that all filters found under config/ have a jail def testReadStockJailFilterComplete(self): - jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) + jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG) self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine # grab all filter names @@ -586,7 +588,7 @@ class JailsReaderTest(LogCaptureTestCase): def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises # on users' systems when enabling shipped jails - jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert(allow_no_files=True) @@ -697,7 +699,7 @@ action = testaction1[actname=test1] filter = testfilter1 """) jailfd.close() - jails = JailsReader(basedir=basedir, share_config=self.__share_cfg) + jails = JailsReader(basedir=basedir, share_config={}) self.assertTrue(jails.read()) self.assertTrue(jails.getOptions()) comm_commands = jails.convert(allow_no_files=True) diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 3cac0540..a8a71723 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -248,55 +248,4 @@ class FailmanagerComplex(unittest.TestCase): self.assertEqual(str(ip), '127.0.255.255') elif i == 65535: self.assertEqual(str(ip), '127.1.0.0') - - def testFailuresMemLeak1(self): - # use factor (divisor) instead unittest.F2B.SkipIfFast() : - modeDiv = 1 - if unittest.F2B.fast: # pragma: no cover - modeDiv = 10 - self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv) - import gc - gc.collect() - timestamp = 1167606999.0 - ticktime = timestamp-(1000 // modeDiv) - for i, ip in self._ip_range(30000 // modeDiv): - t = FailTicket(ip, ticktime, list({'match': i})) - ticktime += 1 - self.__failManager.addFailure(t) - if i % (500 // modeDiv) == 0: - self.__failManager.cleanup(timestamp) - if i % (888 // modeDiv) == 0: - timestamp += (1000 // modeDiv) - ticktime = timestamp-(1000 // modeDiv) - self.assertFalse(gc.collect()) - self.__failManager.cleanup(timestamp) - ticktime -= timestamp - self.__failManager.getMaxTime() + 1 - self.assertEqual(self.__failManager.size(), ticktime if ticktime > 0 else 0) - self.assertFalse(gc.collect()) - - def testFailuresMemLeak2(self): - # use factor (divisor) instead unittest.F2B.SkipIfFast() : - modeDiv = 1 - if unittest.F2B.fast: # pragma: no cover - modeDiv = 10 - self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv) - import gc - gc.collect() - timestamp = 1167606999.0 - ticktime = timestamp-(1000 // modeDiv) - for i, ip in self._ip_range(10000 // modeDiv): - for j in range(0, 5): - t = FailTicket(ip, ticktime, list({'match': i})) - ticktime += 1 - self.__failManager.addFailure(t) - if i % (500 // modeDiv) == 0: - self.__failManager.cleanup(timestamp) - if i % (888 // modeDiv) == 0 or i % (1500 // modeDiv) == 0: - timestamp += (1000 // modeDiv) - ticktime = timestamp-(1000 // modeDiv) - self.assertFalse(gc.collect()) - timestamp += 20000 - self.__failManager.cleanup(timestamp) - self.assertEqual(self.__failManager.size(), 0) - self.assertFalse(gc.collect()) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c4dd716d..c9eddca1 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -80,6 +80,7 @@ def _getSysPythonVersion(): class SetupTest(unittest.TestCase): def setUp(self): + unittest.F2B.SkipIfFast() setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None if not self.setup and sys.version_info >= (2,7): # pragma: no cover - running not out of the source diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 9e6c0ee7..2ed77554 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -64,7 +64,8 @@ def testSampleRegexsFactory(name): def testFilter(self): # Check filter exists - filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR) + filterConf = FilterReader(name, "jail", {}, + basedir=CONFIG_DIR, share_config=unittest.F2B.share_config) self.assertEqual(filterConf.getFile(), name) self.assertEqual(filterConf.getJailName(), "jail") filterConf.read() diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 87ca4289..96734262 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -47,6 +47,7 @@ except ImportError: # pragma: no cover filtersystemd = None TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") +FAST_BACKEND = "polling" class TestServer(Server): @@ -72,7 +73,7 @@ class TransmitterBase(unittest.TestCase): self.tmp_files.append(pidfile_name) self.server.start(sock_name, pidfile_name, force=False) self.jailName = "TestJail1" - self.server.addJail(self.jailName, "auto") + self.server.addJail(self.jailName, FAST_BACKEND) def tearDown(self): """Call after every test case.""" @@ -195,7 +196,7 @@ class Transmitter(TransmitterBase): self.setGetTest("dbpurgeage", "600", 600) self.setGetTestNOK("dbpurgeage", "LIZARD") # the same file name (again with jails / not changed): - self.server.addJail(self.jailName, "auto") + self.server.addJail(self.jailName, FAST_BACKEND) self.setGetTest("dbfile", tmpFilename) self.server.delJail(self.jailName) @@ -213,7 +214,7 @@ class Transmitter(TransmitterBase): ["get", "dbpurgeage"]), (0, None)) # the same (again with jails / not changed): - self.server.addJail(self.jailName, "auto") + self.server.addJail(self.jailName, FAST_BACKEND) self.assertEqual(self.transm.proceed( ["set", "dbfile", "None"]), (0, None)) @@ -252,7 +253,7 @@ class Transmitter(TransmitterBase): self.assertTrue(self.jailName not in self.server._Server__jails) def testStartStopAllJail(self): - self.server.addJail("TestJail2", "auto") + self.server.addJail("TestJail2", FAST_BACKEND) self.assertEqual( self.transm.proceed(["start", self.jailName]), (0, None)) self.assertEqual( @@ -497,7 +498,7 @@ class Transmitter(TransmitterBase): jails = [self.jailName] self.assertEqual(self.transm.proceed(["status"]), (0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))])) - self.server.addJail("TestJail2", "auto") + self.server.addJail("TestJail2", FAST_BACKEND) jails.append("TestJail2") self.assertEqual(self.transm.proceed(["status"]), (0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))])) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index c33c2b19..0cac9df5 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -59,6 +59,7 @@ class F2B(optparse.Values): if self.fast: self.memory_db = True self.no_gamin = True + self.__dict__['share_config'] = {} def SkipIfFast(self): pass def SkipIfNoNetwork(self):