From 2a3790f8e8c45a6867e1f68be68272ddaa4dc9e8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Nov 2014 13:24:54 -0500 Subject: [PATCH 001/140] use iptables-allports for recidive --- ChangeLog | 3 +++ config/jail.conf | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0e107735..c4fc1a44 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee + * recidive uses iptables-allports banaction by default now. + Avoids problems with iptables versions not understanding 'all' for + protocols and ports - New Features: diff --git a/config/jail.conf b/config/jail.conf index d119d229..d2949023 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -655,8 +655,7 @@ maxretry = 5 [recidive] logpath = /var/log/fail2ban.log -port = all -protocol = all +banaction = iptables-allports bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 From 6c34cf2e1e65377106d501e7389aa324108b1998 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 17:52:45 -0500 Subject: [PATCH 002/140] Add .idea dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a8942050..780ecfb5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ htmlcov *.bak __pycache__ .vagrant/ +.idea/ From 2d7429c47cfef27f89818e665c6b7634d9f6d140 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 18:05:19 -0500 Subject: [PATCH 003/140] Add 'Client host rejected error message' regex Not sure if it was reworded (using Postfix 2.6) or a slightly different error, but I only have "Client host rejected: cannot find your hostname" --- ChangeLog | 1 + config/filter.d/postfix.conf | 1 + fail2ban/tests/files/logs/postfix | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index c80dac5a..162071e3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive + * filters.d/postfix.conf - add 'Client host rejected error message' failregex - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index a7a05e47..a994d772 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -13,6 +13,7 @@ before = common.conf _daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index ccf2f8bc..fff25bb9 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -20,3 +20,6 @@ Dec 25 02:35:54 platypus postfix/smtpd[9144]: improper command pipelining after # failJSON: { "time": "2004-12-18T02:05:46", "match": true , "host": "216.245.198.245" } Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after NOOP from unknown[216.245.198.245] + +# failJSON: { "time": "2014-12-21T21:17:29", "match": true , "host": "93.184.216.34" } +Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.1 Client host rejected: cannot find your hostname, [93.184.216.34]; from= to= proto=ESMTP helo= From e6ffa2e4a19dbab4782d39cb066d5d1f196ab811 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 18:10:19 -0500 Subject: [PATCH 004/140] Update year in postfix logs test file --- fail2ban/tests/files/logs/postfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index fff25bb9..ee8720f8 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -21,5 +21,5 @@ Dec 25 02:35:54 platypus postfix/smtpd[9144]: improper command pipelining after # failJSON: { "time": "2004-12-18T02:05:46", "match": true , "host": "216.245.198.245" } Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after NOOP from unknown[216.245.198.245] -# failJSON: { "time": "2014-12-21T21:17:29", "match": true , "host": "93.184.216.34" } +# failJSON: { "time": "2004-12-21T21:17:29", "match": true , "host": "93.184.216.34" } Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.1 Client host rejected: cannot find your hostname, [93.184.216.34]; from= to= proto=ESMTP helo= From fe72a5585c3a9970036b15d7957e8d064d9bdefa Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 19:06:17 -0500 Subject: [PATCH 005/140] Create Jail for Postfix based on RBL Use RBL blocks to ban addresses, unique Jail so maxretry can be set to 1 (vs postfix.conf) --- ChangeLog | 2 ++ config/filter.d/postfix-rbl.conf | 19 +++++++++++++++++++ config/jail.conf | 7 +++++++ fail2ban/tests/files/logs/postfix-rbl | 2 ++ 4 files changed, 30 insertions(+) create mode 100644 config/filter.d/postfix-rbl.conf create mode 100644 fail2ban/tests/files/logs/postfix-rbl diff --git a/ChangeLog b/ChangeLog index c80dac5a..b9d60f75 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filter.d/postfix-sasl.conf - failregex is now case insensitive - New Features: + - New filter: + - postfix-rbl Thanks Lee Clemens - New interpolation feature for config readers - `%(known/parameter)s`. (means last known option with name `parameter`). This interpolation makes possible to extend a stock filter or jail regexp in .local file diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf new file mode 100644 index 00000000..4b572241 --- /dev/null +++ b/config/filter.d/postfix-rbl.conf @@ -0,0 +1,19 @@ +# Fail2Ban filter for Postfix's RBL based Blocked hosts +# +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] + +_daemon = postfix/smtpd + +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using \S+; Blocked.* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ + +ignoreregex = + +# Author: Lee Clemens diff --git a/config/jail.conf b/config/jail.conf index 6a95aa12..5544ba6d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -465,6 +465,13 @@ port = smtp,465,submission logpath = %(postfix_log)s +[postfix-rbl] + +port = smtp,465,submission +logpath = %(syslog_mail)s +maxretry = 1 + + [sendmail-auth] port = submission,465,smtp diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl new file mode 100644 index 00000000..fd420fa7 --- /dev/null +++ b/fail2ban/tests/files/logs/postfix-rbl @@ -0,0 +1,2 @@ +# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" } +Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from= to= proto=ESMTP helo= From 0f48cf42844c92120854a05dc47dba6ba3ae8d82 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Tue, 30 Dec 2014 19:14:39 -0500 Subject: [PATCH 006/140] loosen up regex for spamhaus (spamcop says "Blocked" as part of url) --- config/filter.d/postfix-rbl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf index 4b572241..05a8bbc7 100644 --- a/config/filter.d/postfix-rbl.conf +++ b/config/filter.d/postfix-rbl.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = postfix/smtpd -failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using \S+; Blocked.* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ignoreregex = From c2bb3253ad3266d9e86eea549333befcd5205b95 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 05:27:18 -0500 Subject: [PATCH 007/140] Update ChangeLog --- ChangeLog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index df52d48f..cc37ee9f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -317,7 +317,7 @@ some obscure corner of the Internet. Many thanks to our contributors for this release Daniel Black, Yaroslav Halchenko, Steven Hiscocks, Mark McKinstry, Andy Fragen, Orion Poplawski, Alexander Dietrich, JP Espinosa, Jamyn Shanley, Beau Raines, François -Boulogne and others who have helped on IRC and mailing list, logged issues +Boulogne, Donald Yandt and others who have helped on IRC and mailing list, logged issues and bug requests. - IMPORTANT incompatible changes: @@ -422,7 +422,8 @@ and bug requests. * filter.d/uwimap-auth - filter for uwimap-auth IMAP/POP server Steven Hiscocks and Daniel Black * filter.d/selinux-{common,ssh} -- add SELinux date and ssh filter - + Donald Yandt + * action.d/firewallcmd-{multiport,allports}.conf --added two new actions - Enhancements: François Boulogne and Frédéric * filter.d/lighttpd - auth regexs for lighttpd-1.4.31 From 591e44475347845e6c27cada89bb3225f075563f Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 05:32:06 -0500 Subject: [PATCH 008/140] Create firewallcmd-allports.conf --- config/action.d/firewallcmd-allports.conf | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 config/action.d/firewallcmd-allports.conf diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf new file mode 100644 index 00000000..a478f25c --- /dev/null +++ b/config/action.d/firewallcmd-allports.conf @@ -0,0 +1,52 @@ +# Fail2Ban configuration file +# +# Author: Donald Yandt +# Because of the --remove-rules in stop this action requires firewalld-0.3.8+ + + +[INCLUDES] + +before = iptables-blocktype.conf + +actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- + firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN + firewall-cmd --direct --add-rule ipv4 filter 0 -j f2b- + +actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -j f2b- + firewall-cmd --direct --remove-rules ipv4 filter f2b- + firewall-cmd --direct --remove-chain ipv4 filter f2b- + + +# Note: uses regular expression word boundaries '\b' +# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-recidive\b' + +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-\b' + +actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j + +actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b- 0 -s -j + +[Init] + +# Default name of the chain +# +name = default + +chain = INPUT_direct + +# DEV NOTES: +# +# Author: Donald Yandt +# Uses "FirewallD" instead of the "iptables daemon". +# +# +# Output: + +# actionstart: +# $ firewall-cmd --direct --add-chain ipv4 filter f2b-recidive +# success +# $ firewall-cmd --direct --add-rule ipv4 filter f2b-recidive 1000 -j RETURN +# success +# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -j f2b-recidive +# success + From 50e5fd9ed71164aa545a4757cd72ded5c5aa86d4 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 05:32:41 -0500 Subject: [PATCH 009/140] Create firewallcmd-multiport.conf --- config/action.d/firewallcmd-multiport.conf | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 config/action.d/firewallcmd-multiport.conf diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf new file mode 100644 index 00000000..5f800423 --- /dev/null +++ b/config/action.d/firewallcmd-multiport.conf @@ -0,0 +1,61 @@ +# Fail2Ban configuration file +# +# Author: Donald Yandt +# Because of the --remove-rules in stop this action requires firewalld-0.3.8+ + +[INCLUDES] + +before = iptables-blocktype.conf + +[Definition] + +actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- + firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN + firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p -m multiport --dports -j f2b- + +actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --remove-rules ipv4 filter f2b- + firewall-cmd --direct --remove-chain ipv4 filter f2b- + +# Note: uses regular expression word boundaries '\b' +# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-apache-modsecurity\b' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-\b' + +actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j + +actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b- 0 -s -j + +[Init] + +# Default name of the chain +name = default + +chain = INPUT_direct + +port = 1:65535 + +# Option: protocol +# Values: [ tcp | udp | icmp | all ] + +protocol = tcp + + + +# DEV NOTES: +# +# Author: Donald Yandt +# Uses "FirewallD" instead of the "iptables daemon". +# +# +# Output: +# actionstart: +# $ firewall-cmd --direct --add-chain ipv4 filter f2b-apache-modsecurity +# success +# $ firewall-cmd --direct --add-rule ipv4 filter f2b-apache-modsecurity 1000 -j RETURN +# success +# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 80,443 -j f2b-apache-modsecurity +# success +# actioncheck: +# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | grep -q '\bf2b-apache-modsecurity\b' +# f2b-apache-modsecurity + From 1dfd9941df3fd0e278de20b6fdf2ee1aec87af8b Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 05:33:20 -0500 Subject: [PATCH 010/140] Update THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 61e99466..1185f6b9 100644 --- a/THANKS +++ b/THANKS @@ -33,6 +33,7 @@ Daniel B. Daniel Black David Nutter Derek Atkins +Donald Yandt Eric Gerbier Enrico Labedzki Eugene Hopkinson (SlowRiot) From a47001ea0ea7b6f3d2a5b1aa9cef13022deb86a0 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 12:41:31 -0500 Subject: [PATCH 011/140] Updated ChangeLog --- ChangeLog | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index cc37ee9f..b5593554 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released (opposite to simply set failregex/ignoreregex that overwrites it), see gh-867. - Monit config for fail2ban in /files/monit + - New actions: + - action.d/firewallcmd-multiport Thanks Donald Yandt + - action.d/firewallcmd-multiport Thanks Donald Yandt - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 @@ -422,8 +425,7 @@ and bug requests. * filter.d/uwimap-auth - filter for uwimap-auth IMAP/POP server Steven Hiscocks and Daniel Black * filter.d/selinux-{common,ssh} -- add SELinux date and ssh filter - Donald Yandt - * action.d/firewallcmd-{multiport,allports}.conf --added two new actions + - Enhancements: François Boulogne and Frédéric * filter.d/lighttpd - auth regexs for lighttpd-1.4.31 From 9f91cb2fd8bcbbfb4b8892cce527519ceb6a54c2 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 12:44:34 -0500 Subject: [PATCH 012/140] Created firewallcmd-allports --- config/action.d/firewallcmd-allports.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf index a478f25c..c0c378a4 100644 --- a/config/action.d/firewallcmd-allports.conf +++ b/config/action.d/firewallcmd-allports.conf @@ -17,10 +17,10 @@ actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -j f2b- -# Note: uses regular expression word boundaries '\b' -# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-recidive\b' +# Note: uses regular expression whitespaces '\s' & end of line '$' +# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\sf2b-recidive$' -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-\b' +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\sf2b-$' actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j From 7eed55266b7fac7c997a06d6770491bc0aabedd2 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 12:46:48 -0500 Subject: [PATCH 013/140] Created firewallcmd-multiport --- config/action.d/firewallcmd-multiport.conf | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf index 5f800423..da73f9f1 100644 --- a/config/action.d/firewallcmd-multiport.conf +++ b/config/action.d/firewallcmd-multiport.conf @@ -17,9 +17,10 @@ actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state firewall-cmd --direct --remove-rules ipv4 filter f2b- firewall-cmd --direct --remove-chain ipv4 filter f2b- -# Note: uses regular expression word boundaries '\b' -# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-apache-modsecurity\b' -actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\bf2b-\b' +# Note: uses regular expression whitespaces '\s' & end of line '$' +# Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | grep -q '\sf2b-apache-modsecurity$' + +actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q '\sf2b-$' actionban = firewall-cmd --direct --add-rule ipv4 filter f2b- 0 -s -j @@ -32,8 +33,10 @@ name = default chain = INPUT_direct +# Could also use port numbers separated by a comma. port = 1:65535 + # Option: protocol # Values: [ tcp | udp | icmp | all ] @@ -56,6 +59,6 @@ protocol = tcp # $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 80,443 -j f2b-apache-modsecurity # success # actioncheck: -# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | grep -q '\bf2b-apache-modsecurity\b' +# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | grep -q '\sf2b-apache-modsecurity$' # f2b-apache-modsecurity From 948eec64252ecf7169fd9d282e05a9ad247530d5 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 12:56:17 -0500 Subject: [PATCH 014/140] Upd --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b5593554..d3e42ada 100644 --- a/ChangeLog +++ b/ChangeLog @@ -320,7 +320,7 @@ some obscure corner of the Internet. Many thanks to our contributors for this release Daniel Black, Yaroslav Halchenko, Steven Hiscocks, Mark McKinstry, Andy Fragen, Orion Poplawski, Alexander Dietrich, JP Espinosa, Jamyn Shanley, Beau Raines, François -Boulogne, Donald Yandt and others who have helped on IRC and mailing list, logged issues +Boulogne and others who have helped on IRC and mailing list, logged issues and bug requests. - IMPORTANT incompatible changes: From 74c3d5d96cd7e6406896e5024590f393e4c58493 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 1 Jan 2015 13:26:11 -0500 Subject: [PATCH 015/140] Updated ChangeLog --- ChangeLog | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index d3e42ada..ba39d73e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,9 +31,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released see gh-867. - Monit config for fail2ban in /files/monit - New actions: - - action.d/firewallcmd-multiport Thanks Donald Yandt - - action.d/firewallcmd-multiport Thanks Donald Yandt - + - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be From d0bcacd97be8d2eac24c048d0f58b5b93ee90340 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 2 Jan 2015 02:55:40 -0500 Subject: [PATCH 016/140] Fix strptime thread safety issue strptime thread safety hack-around - http://bugs.python.org/issue7980 --- bin/fail2ban-client | 3 +++ bin/fail2ban-regex | 3 +++ bin/fail2ban-server | 3 +++ 3 files changed, 9 insertions(+) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 866a5287..b91c339c 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -430,6 +430,9 @@ class ServerExecutionException(Exception): pass if __name__ == "__main__": # pragma: no cover - can't test main + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 5644dd37..96aca926 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -474,6 +474,9 @@ class Fail2banRegex(object): if __name__ == "__main__": + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") parser = get_opt_parser() (opts, args) = parser.parse_args() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index ec0c0dbe..dc8d81b4 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -131,6 +131,9 @@ class Fail2banServer: return False if __name__ == "__main__": + from time import strptime + # strptime thread safety hack-around - http://bugs.python.org/issue7980 + strptime("2012", "%Y") server = Fail2banServer() if server.start(sys.argv): sys.exit(0) From b26725f654f2c58a219367f5221da0c148a5cd55 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 13:45:06 -0500 Subject: [PATCH 017/140] Move strptime workaround to fail2ban/__init__.py --- bin/fail2ban-client | 3 --- bin/fail2ban-regex | 3 --- bin/fail2ban-server | 3 --- fail2ban/__init__.py | 4 ++++ 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index b91c339c..866a5287 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -430,9 +430,6 @@ class ServerExecutionException(Exception): pass if __name__ == "__main__": # pragma: no cover - can't test main - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 96aca926..5644dd37 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -474,9 +474,6 @@ class Fail2banRegex(object): if __name__ == "__main__": - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") parser = get_opt_parser() (opts, args) = parser.parse_args() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index dc8d81b4..ec0c0dbe 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -131,9 +131,6 @@ class Fail2banServer: return False if __name__ == "__main__": - from time import strptime - # strptime thread safety hack-around - http://bugs.python.org/issue7980 - strptime("2012", "%Y") server = Fail2banServer() if server.start(sys.argv): sys.exit(0) diff --git a/fail2ban/__init__.py b/fail2ban/__init__.py index cbaaf757..b7906099 100644 --- a/fail2ban/__init__.py +++ b/fail2ban/__init__.py @@ -68,3 +68,7 @@ logging.notice = _root_notice # add NOTICE to the priority map of all the levels logging.handlers.SysLogHandler.priority_map['NOTICE'] = 'notice' + +from time import strptime +# strptime thread safety hack-around - http://bugs.python.org/issue7980 +strptime("2012", "%Y") From 541a747d7949e42844c2637d8e94cf92aedf8075 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 15:19:58 -0500 Subject: [PATCH 018/140] Update Changelog with strptime fix --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index c80dac5a..5bf5a084 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive + * fail2ban/__init__.py - Added strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From 2f360ce44724a570f18e20b79f528e1ea947606e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 15:26:34 -0500 Subject: [PATCH 019/140] Update Changelog with strptime fix --- ChangeLog | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e686adb..040ff48c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,11 +21,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive -<<<<<<< HEAD - * fail2ban/__init__.py - Added strptime thread safety hack-around -======= * filters.d/postfix.conf - add 'Client host rejected error message' failregex ->>>>>>> 8fc32bb33f6f8c43fa107f1a6fc3a71bf000a767 + * fail2ban/__init__.py - Added strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From 4714028c69412018498f469837238f0f7b793f53 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 3 Jan 2015 16:16:23 -0500 Subject: [PATCH 020/140] Change case and tense for consistency --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 040ff48c..73a3693b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,7 +22,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released details. Thanks bes.internal * filter.d/postfix-sasl.conf - failregex is now case insensitive * filters.d/postfix.conf - add 'Client host rejected error message' failregex - * fail2ban/__init__.py - Added strptime thread safety hack-around + * fail2ban/__init__.py - add strptime thread safety hack-around - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From d7b7f4bc91be34e80b48c05c86913fc0d4d168d4 Mon Sep 17 00:00:00 2001 From: TorontoMedia Date: Thu, 8 Jan 2015 21:06:43 -0500 Subject: [PATCH 021/140] Update firewallcmd-allports.conf --- config/action.d/firewallcmd-allports.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf index c0c378a4..ce83164b 100644 --- a/config/action.d/firewallcmd-allports.conf +++ b/config/action.d/firewallcmd-allports.conf @@ -8,6 +8,8 @@ before = iptables-blocktype.conf +[Definition] + actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN firewall-cmd --direct --add-rule ipv4 filter 0 -j f2b- From 083031524d9005fc5a39e9837aaaa545b2a2a2fe Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 21:14:37 -0500 Subject: [PATCH 022/140] BF: adding missing Definition section header to firewallcmd-allports --- config/action.d/firewallcmd-allports.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf index c0c378a4..ce83164b 100644 --- a/config/action.d/firewallcmd-allports.conf +++ b/config/action.d/firewallcmd-allports.conf @@ -8,6 +8,8 @@ before = iptables-blocktype.conf +[Definition] + actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN firewall-cmd --direct --add-rule ipv4 filter 0 -j f2b- From fdd93d1475d595f7af8614307208dac60ee9bb40 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 21:48:50 -0500 Subject: [PATCH 023/140] ENH: unittest to catch actions without Init or Definition section and all must have actionban at least --- fail2ban/tests/clientreadertestcase.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 4f77a698..a94fad4e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -459,6 +459,22 @@ class JailsReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction")) if STOCK: + def testReadStockActionConf(self): + for actionConfig in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')): + actionName = os.path.basename(actionConfig).replace('.conf', '') + actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR) + self.assertTrue(actionReader.read()) + actionReader.getOptions({}) # populate _opts + if not actionName.endswith('-common'): + self.assertTrue('Definition' in actionReader.sections(), + msg="Action file %r is lacking [Definition] section" % actionConfig) + # all must have some actionban defined + self.assertTrue(actionReader._opts.get('actionban', '').strip(), + msg="Action file %r is lacking actionban" % actionConfig) + self.assertTrue('Init' in actionReader.sections(), + msg="Action file %r is lacking [Init] section" % actionConfig) + + def testReadStockJailConf(self): jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine From 12e3cca3f2e1b28228463f59a65e1b5a38114974 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Jan 2015 10:28:53 +0100 Subject: [PATCH 024/140] port[s] typo fixed in jail.conf/nginx-http-auth, issue gh-913 --- ChangeLog | 1 + config/jail.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b4cacb35..0a901547 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: + * port[s] typo in jail.conf/nginx-http-auth gh-913. Thanks Frederik Wagner (fnerdwq) * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee diff --git a/config/jail.conf b/config/jail.conf index 5544ba6d..70104878 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -291,7 +291,7 @@ maxretry = 1 [nginx-http-auth] -ports = http,https +port = http,https logpath = %(nginx_error_log)s From b04a51246f8fa4d1c87d34a3bc6b31369c968e1f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 11:32:15 +0100 Subject: [PATCH 025/140] infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907 --- ChangeLog | 2 ++ fail2ban/server/action.py | 22 +++++++++------------- fail2ban/tests/actiontestcase.py | 2 ++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0a901547..c24c9dc8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: + * infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907. + Thanks TonyThompson * port[s] typo in jail.conf/nginx-http-auth gh-913. Thanks Frederik Wagner (fnerdwq) * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index da7517f6..c69ba88f 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -394,20 +394,16 @@ class CommandAction(ActionBase): # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False - elif found_tag in cls._escapedTags: - # Escaped so won't match + if found_tag in cls._escapedTags or not tags.has_key(found_tag): + # Escaped or missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) continue - else: - if tags.has_key(found_tag): - value = value.replace('<%s>' % found_tag , tags[found_tag]) - #logSys.log(5, 'value now: %s' % value) - done.append(found_tag) - m = t.search(value, m.start()) - else: - # Missing tags are ok so we just continue on searching. - # cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.start() + 1) + value = value.replace('<%s>' % found_tag , tags[found_tag]) + #logSys.log(5, 'value now: %s' % value) + done.append(found_tag) + m = t.search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) tags[tag] = value return tags diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 5a58149f..36e0ddb8 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -59,6 +59,8 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(CommandAction.substituteRecursiveTags({'A': ''}), {'A': ''}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + # Escaped tags should be ignored + self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Multiple stuff on same line is ok self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), { 'failregex': "to=pokie fromip= evilperson=pokie", From 607af36ad3ac9716d8820572538e4071babe1a5a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 14:08:30 +0100 Subject: [PATCH 026/140] workaround for the "Bad file descriptor" issue on Python 2.7, gh-161 : asyncore.loop() using poll by the way, prevents to write "'build/bdist.linux-x86_64' does not exist -- can't clean it" into stderr; --- fail2ban/server/asyncserver.py | 7 +++++-- fail2ban/tests/misctestcase.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index a54d41a1..c99cdffe 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -149,8 +149,11 @@ class AsyncServer(asyncore.dispatcher): self.__init = True # TODO Add try..catch # There's a bug report for Python 2.6/3.0 that use_poll=True yields some 2.5 incompatibilities: - logSys.debug("Detected Python 2.6 or greater. asyncore.loop() not using poll") - asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 + if sys.version_info >= (2, 7) and sys.version_info < (2, 8): # if python 2.7 ... + logSys.debug("Detected Python 2.7. asyncore.loop() using poll") + asyncore.loop(use_poll=True) # workaround for the "Bad file descriptor" issue on Python 2.7, gh-161 + else: + asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 ## # Stops the communication server. diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 2f69aef0..a682b63f 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -113,7 +113,7 @@ class SetupTest(unittest.TestCase): # clean up shutil.rmtree(tmp) # remove build directory - os.system("%s %s clean --all >/dev/null" + os.system("%s %s clean --all >/dev/null 2>&1" % (sys.executable, self.setup)) class TestsUtilsTest(unittest.TestCase): From 33e9e2174ad98466c6f33dd78ba7baf30f3a8340 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 17:09:13 +0100 Subject: [PATCH 027/140] recursive/embedded version of issue/907; test cases merged from remote-tracking branch 'yarikoptic:enh/embedded_tags' into issue/907 infinite busy loop on _escapedTags match in substituteRecursiveTags gh-907 --- ChangeLog | 3 ++ fail2ban/server/action.py | 61 ++++++++++++++++++-------------- fail2ban/tests/actiontestcase.py | 5 +++ 3 files changed, 42 insertions(+), 27 deletions(-) diff --git a/ChangeLog b/ChangeLog index c24c9dc8..f01ed930 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,9 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: - New filter: - postfix-rbl Thanks Lee Clemens + - New recursive embedded substitution feature added: + - `<HOST>` becomes `` for PREF=`IPV4`; + - `<HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`; - New interpolation feature for config readers - `%(known/parameter)s`. (means last known option with name `parameter`). This interpolation makes possible to extend a stock filter or jail regexp in .local file diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index c69ba88f..bfd2adc2 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -378,34 +378,41 @@ class CommandAction(ActionBase): Dictionary of tags(keys) and their values, with tags within the values recursively replaced. """ - t = re.compile(r'<([^ >]+)>') - for tag in tags.iterkeys(): - if tag in cls._escapedTags: - # Escaped so won't match - continue - value = str(tags[tag]) - m = t.search(value) - done = [] - #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) - while m: - found_tag = m.group(1) - #logSys.log(5, 'found: %s' % found_tag) - if found_tag == tag or found_tag in done: - # recursive definitions are bad - #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) - return False - if found_tag in cls._escapedTags or not tags.has_key(found_tag): - # Escaped or missing tags - just continue on searching after end of match - # Missing tags are ok - cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.end()) + t = re.compile(r'<([^ <>]+)>') + while True: + repFlag = False + for tag in tags.iterkeys(): + if tag in cls._escapedTags: + # Escaped so won't match continue - value = value.replace('<%s>' % found_tag , tags[found_tag]) - #logSys.log(5, 'value now: %s' % value) - done.append(found_tag) - m = t.search(value, m.start()) - #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - tags[tag] = value + value = str(tags[tag]) + m = t.search(value) + done = [] + #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) + while m: + found_tag = m.group(1) + #logSys.log(5, 'found: %s' % found_tag) + if found_tag == tag or found_tag in done: + # recursive definitions are bad + #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) + return False + if found_tag in cls._escapedTags or not tags.has_key(found_tag): + # Escaped or missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) + continue + value = value.replace('<%s>' % found_tag , tags[found_tag]) + #logSys.log(5, 'value now: %s' % value) + done.append(found_tag) + m = t.search(value, m.start()) + #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) + # if was substituted, check again later: + if tags[tag] != value: + repFlag = True + tags[tag] = value + if not repFlag: + break return tags @staticmethod diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 36e0ddb8..7cd0cd4d 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -73,6 +73,11 @@ class CommandActionTest(LogCaptureTestCase): 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', }) + # obscure embedded case + self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), + {'A': '', 'PREF': 'IPV4'}) + self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), + {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self): aInfo = { From 6b42878b8cea7a3d3ea8ad261a0e51f480d051e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Jan 2015 17:31:17 +0100 Subject: [PATCH 028/140] better recognition of embedded-recursive substitution to repeat interpolation --- fail2ban/server/action.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index bfd2adc2..70b3eb7e 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -407,9 +407,11 @@ class CommandAction(ActionBase): done.append(found_tag) m = t.search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - # if was substituted, check again later: + # was substituted? if tags[tag] != value: - repFlag = True + # check again later if embedded-recursive substitution: + if value.startswith('<') and value.endswith('>'): + repFlag = True tags[tag] = value if not repFlag: break From d0b932aaca91cb044861a3e61496c6c06b03c263 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 21 Jan 2015 09:44:55 +0100 Subject: [PATCH 029/140] code review + more test cases (embedded replace in a string) --- fail2ban/server/action.py | 8 ++++++-- fail2ban/tests/actiontestcase.py | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 70b3eb7e..8e0cd97f 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -362,6 +362,7 @@ class CommandAction(ActionBase): @classmethod def substituteRecursiveTags(cls, tags): """Sort out tag definitions within other tags. + Since v.0.9.2 supports embedded interpolation (see test cases for examples). so: becomes: a = 3 a = 3 @@ -379,13 +380,16 @@ class CommandAction(ActionBase): within the values recursively replaced. """ t = re.compile(r'<([^ <>]+)>') + # repeat substitution while embedded-recursive (repFlag is True) while True: repFlag = False + # substitute each value: for tag in tags.iterkeys(): if tag in cls._escapedTags: # Escaped so won't match continue value = str(tags[tag]) + # search and replace all tags within value, that can be interpolated using other tags: m = t.search(value) done = [] #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) @@ -409,8 +413,8 @@ class CommandAction(ActionBase): #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) # was substituted? if tags[tag] != value: - # check again later if embedded-recursive substitution: - if value.startswith('<') and value.endswith('>'): + # check still contains any tag - should be repeated (possible embedded-recursive substitution): + if t.search(value): repFlag = True tags[tag] = value if not repFlag: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 7cd0cd4d..2e826a8e 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -78,6 +78,9 @@ class CommandActionTest(LogCaptureTestCase): {'A': '', 'PREF': 'IPV4'}) self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) + # more embedded within a string and two interpolations + self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), + {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self): aInfo = { From 40eb7a40f73ff98aca4ddc227045fa7c54b7d74e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 13:54:00 -0500 Subject: [PATCH 030/140] Add pypy3 Travis Support --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index bd2d294c..0a4486a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ python: - "3.3" - "3.4" - "pypy" + - "pypy3" before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: From d01099f3478e330ccab076c9f2cbb52d66baf599 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 14:43:00 -0500 Subject: [PATCH 031/140] Use travis_retry for commands which use the internet --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd2d294c..ead9ef46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ python: - "3.4" - "pypy" before_install: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi install: - - pip install pyinotify - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi + - travis_retry pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi # test installation From 625bcef8a1be6bbcdd4d4522aafe46d917d8e472 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:12:04 -0500 Subject: [PATCH 032/140] Add extended info to status output using Cyrmu --- .travis.yml | 2 + ChangeLog | 6 +- fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 13 +++- fail2ban/server/banmanager.py | 112 +++++++++++++++++++++++++++ fail2ban/server/jail.py | 9 +++ fail2ban/server/jailthread.py | 6 ++ fail2ban/server/server.py | 5 +- fail2ban/server/transmitter.py | 7 +- fail2ban/tests/banmanagertestcase.py | 50 ++++++++++-- fail2ban/tests/servertestcase.py | 25 ++++++ fail2ban/tests/utils.py | 1 + 12 files changed, 227 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd2d294c..ce14e0ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: - pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi script: diff --git a/ChangeLog b/ChangeLog index f01ed930..3013dac8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,7 +41,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Monit config for fail2ban in /files/monit - New actions: - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt - + - New status commands: + - fail2ban-client status extended + - prints Cymru data (ASN, Country RIR) per banned IP + - Requires dnspython or dnspython3 + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 9a6ee675..dc7a7c6b 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,7 @@ protocol = [ ["start ", "starts the jail "], ["stop ", "stops the jail . The jail is removed"], ["status ", "gets the current status of "], +["status extended", "gets the current status of with extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], ["set addignoreip ", "adds to the ignore list of "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a212aaf1..f4d5c18b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -372,9 +372,20 @@ class Actions(JailThread, Mapping): @property def status(self): - """Status of active bans, and total ban counts. + """Status of current and total ban counts and current banned IP list. """ ret = [("Currently banned", self.__banManager.size()), ("Total banned", self.__banManager.getBanTotal()), ("Banned IP list", self.__banManager.getBanList())] return ret + + @property + def statusExtended(self): + """Jail status plus banned IPs' ASN, Country and RIR + """ + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + ret = self.status +\ + [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index c21cad45..81ac7c4f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,6 +26,9 @@ __license__ = "GPL" from threading import Lock +import dns.exception +import dns.resolver + from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -118,6 +121,115 @@ class BanManager: finally: self.__lock.release() + ## + # Returns normalized value + # + # @return value or "unknown" if value is None or empty string + + @staticmethod + def handleBlankResult(value): + if value is None or len(value) == 0: + return "unknown" + else: + return value + + ## + # Returns Cymru DNS query information + # + # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs + + def getBanListExtendedCymruInfo(self): + self.__lock.acquire() + return_dict = {"asn": [], "country": [], "rir": []} + try: + for banData in self.__banList: + ip = banData.getIP() + # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # TODO: IPv6 compatibility + reversed_ip = ".".join(reversed(ip.split("."))) + question = "%s.origin.asn.cymru.com" % reversed_ip + try: + answers = dns.resolver.query(question, "TXT") + for rdata in answers: + asn, net, country, rir, changed =\ + [answer.strip("'\" ") for answer in rdata.to_text().split("|")] + asn = self.handleBlankResult(asn) + country = self.handleBlankResult(country) + rir = self.handleBlankResult(rir) + return_dict["asn"].append(self.handleBlankResult(asn)) + return_dict["country"].append(self.handleBlankResult(country)) + return_dict["rir"].append(self.handleBlankResult(rir)) + except dns.resolver.NXDOMAIN: + return_dict["asn"].append("nxdomain") + return_dict["country"].append("nxdomain") + return_dict["rir"].append("nxdomain") + except dns.exception.DNSException as dnse: + logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question) + logSys.exception(dnse) + except Exception as e: + logSys.error("Unhandled Exception querying Cymru for %s TXT" % question) + logSys.exception(e) + except Exception as e: + logSys.error("Failure looking up extended Cymru info") + logSys.exception(e) + finally: + self.__lock.release() + return return_dict + + ## + # Returns list of Banned ASNs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned ASNs + + def geBanListExtendedASN(self, cymru_info): + self.__lock.acquire() + try: + return [asn for asn in cymru_info["asn"]] + except Exception as e: + logSys.error("Failed to lookup ASN") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned Countries from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned Countries + + def geBanListExtendedCountry(self, cymru_info): + self.__lock.acquire() + try: + return [country for country in cymru_info["country"]] + except Exception as e: + logSys.error("Failed to lookup Country") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned RIRs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned RIRs + + def geBanListExtendedRIR(self, cymru_info): + self.__lock.acquire() + try: + return [rir for rir in cymru_info["rir"]] + except Exception as e: + logSys.error("Failed to lookup RIR") + logSys.exception(e) + return [] + finally: + self.__lock.release() + ## # Create a ban ticket. # diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index e48e5d7b..df28945e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -183,6 +183,15 @@ class Jail: ("Actions", self.actions.status), ] + @property + def statusExtended(self): + """The extended status of the jail. + """ + return [ + ("Filter", self.filter.status), + ("Actions", self.actions.statusExtended), + ] + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 2627cebf..123a2891 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -72,6 +72,12 @@ class JailThread(Thread): """ pass + @abstractproperty + def statusExtended(self): # pragma: no cover - abstract + """Abstract - Should provide extended status information. + """ + pass + def start(self): """Sets active flag and starts thread. """ diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 02f3423b..c8840608 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -322,7 +322,10 @@ class Server: def statusJail(self, name): return self.__jails[name].status - + + def statusJailExtended(self, name): + return self.__jails[name].statusExtended + # Logging ## diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 39157128..05498646 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -333,5 +333,10 @@ class Transmitter: elif len(command) == 1: name = command[0] return self.__server.statusJail(name) + elif len(command) == 2: + name = command[0] + if command[1] == "extended": + return self.__server.statusJailExtended(name) + else: + raise Exception("Invalid command (invalid status extension)") raise Exception("Invalid command (no status)") - diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 7dcb73a7..d4d106a9 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -30,7 +30,6 @@ from ..server.banmanager import BanManager from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): - def setUp(self): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) @@ -39,19 +38,58 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testAdd(self): self.assertEqual(self.__banManager.size(), 1) - + def testAddDuplicate(self): self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + def testInListOK(self): ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) - + def testInListNOK(self): ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) - + + +class StatusExtendedCymruInfo(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.__ban_ip = "93.184.216.34" + self.__asn = "15133" + self.__country = "EU" + self.__rir = "ripencc" + self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + + def tearDown(self): + """Call after every test case.""" + + def testCymruInfo(self): + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn]) + self.assertEqual(cymru_info["country"], [self.__country]) + self.assertEqual(cymru_info["rir"], [self.__rir]) + + def testCymruInfoASN(self): + self.assertEqual( + self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()), + [self.__asn]) + + def testCymruInfoCountry(self): + self.assertEqual( + self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()), + [self.__country]) + + def testCymruInfoRIR(self): + self.assertEqual( + self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), + [self.__rir]) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index e018ed2f..3bfcc303 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -474,6 +474,27 @@ class Transmitter(TransmitterBase): ) ) + def testJailStatusExtended(self): + self.assertEqual(self.transm.proceed(["status", self.jailName, "extended"]), + (0, + [ + ('Filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('Actions', [ + ('Currently banned', 0), + ('Total banned', 0), + ('Banned IP list', []), + ('Banned ASN list', []), + ('Banned Country list', []), + ('Banned RIR list', [])] + ) + ] + ) + ) + def testAction(self): action = "TestCaseAction" cmdList = [ @@ -601,6 +622,10 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + def testStatusJailExtendedNOK(self): + self.assertEqual( + self.transm.proceed(["status", self.jailName, "INVALID_COMMAND"])[0],1) + def testJournalMatch(self): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b56c0988..42edf369 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,6 +107,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From a8f057eab749d3fbf3974edf73ff9b4b78a32a05 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:27:33 -0500 Subject: [PATCH 033/140] Add `pass` to empty methods --- fail2ban/tests/banmanagertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index d4d106a9..9cf820a3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -38,6 +38,7 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testAdd(self): self.assertEqual(self.__banManager.size(), 1) @@ -68,6 +69,7 @@ class StatusExtendedCymruInfo(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() From a80ff26e151c295bb5cc1661fa5402c0115a8e97 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:40:00 -0500 Subject: [PATCH 034/140] Conditionally test fail2ban-client status extended when dnspython is installed --- fail2ban/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 42edf369..dbe5cee3 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,7 +107,11 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) - tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + try: + import dns + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + except ImportError: + pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From 8fd9d6a770bdfc90d59da6c1493d82e0b1a2752e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:43:58 -0500 Subject: [PATCH 035/140] add dnspython[3] optional dependencies --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8210837e..4915c95b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) +- [dnspython or dnspython3](http://www.dnspython.org/) To install, just do: From 256c8711210b17a63d225610afc03a313cef311d Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:33 -0500 Subject: [PATCH 036/140] conditionally import dnspython --- fail2ban/server/banmanager.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 81ac7c4f..2b2fd39a 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,9 +26,6 @@ __license__ = "GPL" from threading import Lock -import dns.exception -import dns.resolver - from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -139,8 +136,17 @@ class BanManager: # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs def getBanListExtendedCymruInfo(self): - self.__lock.acquire() return_dict = {"asn": [], "country": [], "rir": []} + try: + import dns.exception + import dns.resolver + except ImportError: + logSys.error("dnspython package is required but could not be imported") + return_dict["asn"].append("error") + return_dict["country"].append("error") + return_dict["rir"].append("error") + return return_dict + self.__lock.acquire() try: for banData in self.__banList: ip = banData.getIP() From ce0fe8b907000a02eca1b8edddd897325580a1f6 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:49 -0500 Subject: [PATCH 037/140] test dnspython nxdomain returned --- fail2ban/tests/banmanagertestcase.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 9cf820a3..3f7fe97e 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -63,9 +63,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.__asn = "15133" self.__country = "EU" self.__rir = "ripencc" - self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager.addBanTicket(ticket)) def tearDown(self): """Call after every test case.""" @@ -74,7 +74,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + self.assertDictEqual(cymru_info, {"asn": [self.__asn], + "country": [self.__country], + "rir": [self.__rir]}) else: # Python 2.6 does not support assertDictEqual() self.assertEqual(cymru_info["asn"], [self.__asn]) @@ -95,3 +97,18 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertEqual( self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), [self.__rir]) + + def testCymruInfoNxdomain(self): + ticket = BanTicket("10.0.0.0", 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(ticket)) + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], + "country": [self.__country, "nxdomain"], + "rir": [self.__rir, "nxdomain"]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) + self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) + self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) From 03db257c434e93b80742d33ee68773471d70c617 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:18:09 -0500 Subject: [PATCH 038/140] fix test of new banManager's instance --- fail2ban/tests/banmanagertestcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 3f7fe97e..c4dd42cf 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -104,11 +104,11 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertTrue(self.__banManager.addBanTicket(ticket)) cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], - "country": [self.__country, "nxdomain"], - "rir": [self.__rir, "nxdomain"]}) + self.assertDictEqual(cymru_info, {"asn": ["nxdomain"], + "country": ["nxdomain"], + "rir": ["nxdomain"]}) else: # Python 2.6 does not support assertDictEqual() - self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) - self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) - self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) + self.assertEqual(cymru_info["asn"], ["nxdomain"]) + self.assertEqual(cymru_info["country"], ["nxdomain"]) + self.assertEqual(cymru_info["rir"], ["nxdomain"]) From 13b2aea48a370b84075ca4ce80ea0f7ce9d69491 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 24 Jan 2015 12:22:34 -0500 Subject: [PATCH 039/140] Remove version specific package --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4915c95b..17a78ee5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) -- [dnspython or dnspython3](http://www.dnspython.org/) +- [dnspython](http://www.dnspython.org/) To install, just do: From 60ac0a1a1797a9a10fe7d6b91003571de5c54a5e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:12:04 -0500 Subject: [PATCH 040/140] Add extended info to status output using Cyrmu --- .travis.yml | 2 + ChangeLog | 6 +- fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 13 +++- fail2ban/server/banmanager.py | 112 +++++++++++++++++++++++++++ fail2ban/server/jail.py | 9 +++ fail2ban/server/jailthread.py | 6 ++ fail2ban/server/server.py | 5 +- fail2ban/server/transmitter.py | 7 +- fail2ban/tests/banmanagertestcase.py | 50 ++++++++++-- fail2ban/tests/servertestcase.py | 25 ++++++ fail2ban/tests/utils.py | 1 + 12 files changed, 227 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index ead9ef46..32742761 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi install: - travis_retry pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi script: diff --git a/ChangeLog b/ChangeLog index f01ed930..3013dac8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,7 +41,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Monit config for fail2ban in /files/monit - New actions: - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt - + - New status commands: + - fail2ban-client status extended + - prints Cymru data (ASN, Country RIR) per banned IP + - Requires dnspython or dnspython3 + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 9a6ee675..dc7a7c6b 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,7 @@ protocol = [ ["start ", "starts the jail "], ["stop ", "stops the jail . The jail is removed"], ["status ", "gets the current status of "], +["status extended", "gets the current status of with extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], ["set addignoreip ", "adds to the ignore list of "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a212aaf1..f4d5c18b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -372,9 +372,20 @@ class Actions(JailThread, Mapping): @property def status(self): - """Status of active bans, and total ban counts. + """Status of current and total ban counts and current banned IP list. """ ret = [("Currently banned", self.__banManager.size()), ("Total banned", self.__banManager.getBanTotal()), ("Banned IP list", self.__banManager.getBanList())] return ret + + @property + def statusExtended(self): + """Jail status plus banned IPs' ASN, Country and RIR + """ + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + ret = self.status +\ + [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index c21cad45..81ac7c4f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,6 +26,9 @@ __license__ = "GPL" from threading import Lock +import dns.exception +import dns.resolver + from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -118,6 +121,115 @@ class BanManager: finally: self.__lock.release() + ## + # Returns normalized value + # + # @return value or "unknown" if value is None or empty string + + @staticmethod + def handleBlankResult(value): + if value is None or len(value) == 0: + return "unknown" + else: + return value + + ## + # Returns Cymru DNS query information + # + # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs + + def getBanListExtendedCymruInfo(self): + self.__lock.acquire() + return_dict = {"asn": [], "country": [], "rir": []} + try: + for banData in self.__banList: + ip = banData.getIP() + # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # TODO: IPv6 compatibility + reversed_ip = ".".join(reversed(ip.split("."))) + question = "%s.origin.asn.cymru.com" % reversed_ip + try: + answers = dns.resolver.query(question, "TXT") + for rdata in answers: + asn, net, country, rir, changed =\ + [answer.strip("'\" ") for answer in rdata.to_text().split("|")] + asn = self.handleBlankResult(asn) + country = self.handleBlankResult(country) + rir = self.handleBlankResult(rir) + return_dict["asn"].append(self.handleBlankResult(asn)) + return_dict["country"].append(self.handleBlankResult(country)) + return_dict["rir"].append(self.handleBlankResult(rir)) + except dns.resolver.NXDOMAIN: + return_dict["asn"].append("nxdomain") + return_dict["country"].append("nxdomain") + return_dict["rir"].append("nxdomain") + except dns.exception.DNSException as dnse: + logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question) + logSys.exception(dnse) + except Exception as e: + logSys.error("Unhandled Exception querying Cymru for %s TXT" % question) + logSys.exception(e) + except Exception as e: + logSys.error("Failure looking up extended Cymru info") + logSys.exception(e) + finally: + self.__lock.release() + return return_dict + + ## + # Returns list of Banned ASNs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned ASNs + + def geBanListExtendedASN(self, cymru_info): + self.__lock.acquire() + try: + return [asn for asn in cymru_info["asn"]] + except Exception as e: + logSys.error("Failed to lookup ASN") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned Countries from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned Countries + + def geBanListExtendedCountry(self, cymru_info): + self.__lock.acquire() + try: + return [country for country in cymru_info["country"]] + except Exception as e: + logSys.error("Failed to lookup Country") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned RIRs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned RIRs + + def geBanListExtendedRIR(self, cymru_info): + self.__lock.acquire() + try: + return [rir for rir in cymru_info["rir"]] + except Exception as e: + logSys.error("Failed to lookup RIR") + logSys.exception(e) + return [] + finally: + self.__lock.release() + ## # Create a ban ticket. # diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index e48e5d7b..df28945e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -183,6 +183,15 @@ class Jail: ("Actions", self.actions.status), ] + @property + def statusExtended(self): + """The extended status of the jail. + """ + return [ + ("Filter", self.filter.status), + ("Actions", self.actions.statusExtended), + ] + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 2627cebf..123a2891 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -72,6 +72,12 @@ class JailThread(Thread): """ pass + @abstractproperty + def statusExtended(self): # pragma: no cover - abstract + """Abstract - Should provide extended status information. + """ + pass + def start(self): """Sets active flag and starts thread. """ diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 02f3423b..c8840608 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -322,7 +322,10 @@ class Server: def statusJail(self, name): return self.__jails[name].status - + + def statusJailExtended(self, name): + return self.__jails[name].statusExtended + # Logging ## diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 39157128..05498646 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -333,5 +333,10 @@ class Transmitter: elif len(command) == 1: name = command[0] return self.__server.statusJail(name) + elif len(command) == 2: + name = command[0] + if command[1] == "extended": + return self.__server.statusJailExtended(name) + else: + raise Exception("Invalid command (invalid status extension)") raise Exception("Invalid command (no status)") - diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 7dcb73a7..d4d106a9 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -30,7 +30,6 @@ from ..server.banmanager import BanManager from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): - def setUp(self): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) @@ -39,19 +38,58 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testAdd(self): self.assertEqual(self.__banManager.size(), 1) - + def testAddDuplicate(self): self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + def testInListOK(self): ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) - + def testInListNOK(self): ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) - + + +class StatusExtendedCymruInfo(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.__ban_ip = "93.184.216.34" + self.__asn = "15133" + self.__country = "EU" + self.__rir = "ripencc" + self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + + def tearDown(self): + """Call after every test case.""" + + def testCymruInfo(self): + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn]) + self.assertEqual(cymru_info["country"], [self.__country]) + self.assertEqual(cymru_info["rir"], [self.__rir]) + + def testCymruInfoASN(self): + self.assertEqual( + self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()), + [self.__asn]) + + def testCymruInfoCountry(self): + self.assertEqual( + self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()), + [self.__country]) + + def testCymruInfoRIR(self): + self.assertEqual( + self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), + [self.__rir]) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index e018ed2f..3bfcc303 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -474,6 +474,27 @@ class Transmitter(TransmitterBase): ) ) + def testJailStatusExtended(self): + self.assertEqual(self.transm.proceed(["status", self.jailName, "extended"]), + (0, + [ + ('Filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('Actions', [ + ('Currently banned', 0), + ('Total banned', 0), + ('Banned IP list', []), + ('Banned ASN list', []), + ('Banned Country list', []), + ('Banned RIR list', [])] + ) + ] + ) + ) + def testAction(self): action = "TestCaseAction" cmdList = [ @@ -601,6 +622,10 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + def testStatusJailExtendedNOK(self): + self.assertEqual( + self.transm.proceed(["status", self.jailName, "INVALID_COMMAND"])[0],1) + def testJournalMatch(self): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b56c0988..42edf369 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,6 +107,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From ba699690573da914bdb9c8b5d5e5887303a58ff3 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:27:33 -0500 Subject: [PATCH 041/140] Add `pass` to empty methods --- fail2ban/tests/banmanagertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index d4d106a9..9cf820a3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -38,6 +38,7 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testAdd(self): self.assertEqual(self.__banManager.size(), 1) @@ -68,6 +69,7 @@ class StatusExtendedCymruInfo(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() From 405f363fe87cb798ad6b6cdd36cb5a26e5e487e4 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:40:00 -0500 Subject: [PATCH 042/140] Conditionally test fail2ban-client status extended when dnspython is installed --- fail2ban/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 42edf369..dbe5cee3 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,7 +107,11 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) - tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + try: + import dns + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + except ImportError: + pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From ef3c942808f9472e82854122828f4db39e62d85f Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:43:58 -0500 Subject: [PATCH 043/140] add dnspython[3] optional dependencies --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8210837e..4915c95b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) +- [dnspython or dnspython3](http://www.dnspython.org/) To install, just do: From 07a47179a774e3d56861cee9fc231799e19f7fc9 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:33 -0500 Subject: [PATCH 044/140] conditionally import dnspython --- fail2ban/server/banmanager.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 81ac7c4f..2b2fd39a 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,9 +26,6 @@ __license__ = "GPL" from threading import Lock -import dns.exception -import dns.resolver - from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -139,8 +136,17 @@ class BanManager: # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs def getBanListExtendedCymruInfo(self): - self.__lock.acquire() return_dict = {"asn": [], "country": [], "rir": []} + try: + import dns.exception + import dns.resolver + except ImportError: + logSys.error("dnspython package is required but could not be imported") + return_dict["asn"].append("error") + return_dict["country"].append("error") + return_dict["rir"].append("error") + return return_dict + self.__lock.acquire() try: for banData in self.__banList: ip = banData.getIP() From a0debea56a3efb4609918f39b32497536516fb92 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:49 -0500 Subject: [PATCH 045/140] test dnspython nxdomain returned --- fail2ban/tests/banmanagertestcase.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 9cf820a3..3f7fe97e 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -63,9 +63,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.__asn = "15133" self.__country = "EU" self.__rir = "ripencc" - self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager.addBanTicket(ticket)) def tearDown(self): """Call after every test case.""" @@ -74,7 +74,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + self.assertDictEqual(cymru_info, {"asn": [self.__asn], + "country": [self.__country], + "rir": [self.__rir]}) else: # Python 2.6 does not support assertDictEqual() self.assertEqual(cymru_info["asn"], [self.__asn]) @@ -95,3 +97,18 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertEqual( self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), [self.__rir]) + + def testCymruInfoNxdomain(self): + ticket = BanTicket("10.0.0.0", 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(ticket)) + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], + "country": [self.__country, "nxdomain"], + "rir": [self.__rir, "nxdomain"]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) + self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) + self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) From 735c51adaefd245f215aed5381c7d8f7ca8d86ed Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:18:09 -0500 Subject: [PATCH 046/140] fix test of new banManager's instance --- fail2ban/tests/banmanagertestcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 3f7fe97e..c4dd42cf 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -104,11 +104,11 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertTrue(self.__banManager.addBanTicket(ticket)) cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], - "country": [self.__country, "nxdomain"], - "rir": [self.__rir, "nxdomain"]}) + self.assertDictEqual(cymru_info, {"asn": ["nxdomain"], + "country": ["nxdomain"], + "rir": ["nxdomain"]}) else: # Python 2.6 does not support assertDictEqual() - self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) - self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) - self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) + self.assertEqual(cymru_info["asn"], ["nxdomain"]) + self.assertEqual(cymru_info["country"], ["nxdomain"]) + self.assertEqual(cymru_info["rir"], ["nxdomain"]) From dfe4d02f6540fdb6879500589da798b621ea5ff2 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 24 Jan 2015 12:22:34 -0500 Subject: [PATCH 047/140] Remove version specific package --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4915c95b..17a78ee5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) -- [dnspython or dnspython3](http://www.dnspython.org/) +- [dnspython](http://www.dnspython.org/) To install, just do: From eb76dcd5a00079b39b1b200f3c9f12fdc4c0fa6a Mon Sep 17 00:00:00 2001 From: rumple010 Date: Sun, 25 Jan 2015 23:15:07 -0500 Subject: [PATCH 048/140] add nsupdate action Adds a new action file that uses nsupdate to dynamically update a BIND zone file with a TXT resource record representing a banned IP address. Resource record is deleted from the zone when the ban expires. --- config/action.d/nsupdate.conf | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 config/action.d/nsupdate.conf diff --git a/config/action.d/nsupdate.conf b/config/action.d/nsupdate.conf new file mode 100644 index 00000000..9ecb6f95 --- /dev/null +++ b/config/action.d/nsupdate.conf @@ -0,0 +1,110 @@ +# Fail2Ban configuration file +# +# Author: Andrew St. Jean +# +# Use nsupdate to perform dynamic DNS updates on a BIND zone file. +# One may want to do this to update a local RBL with banned IP addresses. +# +# Options +# +# domain DNS domain that will appear in nsupdate add and delete +# commands. +# +# ttl The time to live (TTL) in seconds of the TXT resource +# record. +# +# rdata Data portion of the TXT resource record. +# +# nsupdatecmd Full path to the nsupdate command. +# +# keyfile Full path to TSIG key file used for authentication between +# nsupdate and BIND. +# +# The ban and unban commands assume nsupdate will authenticate to the BIND +# server using a TSIG key. The full path to the key file must be specified +# in the parameter. Use this command to generate your TSIG key. +# +# dnssec-keygen -a HMAC-MD5 -b 256 -n HOST +# +# Replace with some meaningful name. +# +# This command will generate two files. Specify the .private file in the +# option. Note that the .key file must also be present in the same +# directory for nsupdate to use the key. +# +# Don't forget to add the key and appropriate allow-update or update-policy +# option to your named.conf file. +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = echo | awk -F. '{print "prereq nxrrset "$4"."$3"."$2"."$1". TXT"; print "update add "$4"."$3"."$2"."$1". IN TXT \"\""; print "send"}' | -k + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = echo | awk -F. '{print "update delete "$4"."$3"."$2"."$1"."; print "send"}' | -k + +[Init] + +# Option: domain +# Notes.: DNS domain that nsupdate will update. +# Values: STRING +# +domain = + +# Option: ttl +# Notes.: time to live (TTL) of TXT resource record added by nsupdate. +# Values: NUM +# +ttl = 600 + +# Option: rdata +# Notes.: data portion of the TXT resource record added by nsupdate. +# Values: STRING +# +rdata = Your IP has been banned + +# Option: nsupdatecmd +# Notes.: specifies the full path to the nsupdate program that dynamically +# updates BIND zone files. +# Values: CMD +# +nsupdatecmd = /usr/bin/nsupdate + +# Option: keyfile +# Notes.: specifies the full path to the file containing the +# TSIG key for communicating with BIND. +# Values: STRING +# +keyfile = + From 085d0f72ed1b23e58fe7185b9af099e8a9b4f515 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 26 Jan 2015 09:19:44 -0500 Subject: [PATCH 049/140] ENH: use non-UTC date invocation (without -u) and report offset for localzone (%z) --- ChangeLog | 5 ++++- config/action.d/sendmail-common.conf | 4 ++-- config/action.d/sendmail-whois-ipjailmatches.conf | 2 +- config/action.d/sendmail-whois-ipmatches.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-matches.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- config/action.d/sendmail.conf | 2 +- config/action.d/xarf-login-attack.conf | 2 +- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca2601bc..86dca7f0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,7 +48,10 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be - suitable for manual installations now (thanks Juan Karlo de Guzman) + suitable for manual installations now (thanks Juan Karlo de + Guzman) + * action.d/{sendmail-*,xarg-login-attack}.conf - report local + timezone not UTC time/zone. Closes gh-911 ver. 0.9.1 (2014/10/29) - better, faster, stronger diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf index 26dcb4c8..af0212bd 100644 --- a/config/action.d/sendmail-common.conf +++ b/config/action.d/sendmail-common.conf @@ -15,7 +15,7 @@ after = sendmail-common.local # Values: CMD # actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n @@ -28,7 +28,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` # Values: CMD # actionstop = printf %%b "Subject: [Fail2Ban] : stopped on `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 45b1f312..2de70d85 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index 8193fb04..1390e7a3 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4169e82a..135632ce 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index ed664766..c99d5ebf 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index fc601277..9403a388 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index 46050e11..4b088dc8 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -17,7 +17,7 @@ before = sendmail-common.conf # Values: CMD # actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` - Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` + Date: `LC_TIME=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n Hi,\n diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 6d6a74f0..19b3167f 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -46,7 +46,7 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(di REPORTID=