From 5087b410542daae8d13dd93829e273df08be6ba5 Mon Sep 17 00:00:00 2001 From: blotus Date: Fri, 25 Jan 2013 13:37:22 +0100 Subject: [PATCH 01/45] Escape ' and " in matches tag --- server/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/action.py b/server/action.py index f2614b33..35974c70 100644 --- a/server/action.py +++ b/server/action.py @@ -243,7 +243,7 @@ class Action: return Action.executeCmd(stopCmd) def escapeTag(tag): - for c in '\\#&;`|*?~<>^()[]{}$\n': + for c in '\\#&;`|*?~<>^()[]{}$\n\'"': if c in tag: tag = tag.replace(c, '\\' + c) return tag From 3b0800459b0a21f5c2c55ccd2981198768cea18d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 25 Jan 2013 12:56:00 -0700 Subject: [PATCH 02/45] Initial support for --no-network option for fail2ban-testcases --- fail2ban-testcases | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 0ee2c53c..20d3b226 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -53,6 +53,12 @@ def get_opt_parser(): help="Log level for the logger to use during running tests"), ]) + p.add_options([ + Option('-n', "--no-network", action="store_true", + dest="no_network", + help="Do not run tests that require the network"), + ]) + return p parser = get_opt_parser() @@ -90,6 +96,8 @@ else: stdout.setFormatter(logging.Formatter(' %(message)s')) logSys.addHandler(stdout) +if opts.no_network is None: + opts.no_network = False # # Let know the version @@ -129,11 +137,13 @@ tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) # Filter -tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) +if not opts.no_network: + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) -tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) -tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) +if not opts.no_network: + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector From e4aedfdc00aa4b0a70d244ebc7670c51a4f946f5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 25 Jan 2013 16:01:35 -0500 Subject: [PATCH 03/45] BF: pyinotify - use bitwise op on masks and do not try tracking newly created directories --- server/filterpyinotify.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/filterpyinotify.py b/server/filterpyinotify.py index fdc7256e..a2eea9d7 100644 --- a/server/filterpyinotify.py +++ b/server/filterpyinotify.py @@ -65,7 +65,11 @@ class FilterPyinotify(FileFilter): def callback(self, event): path = event.pathname - if event.mask == pyinotify.IN_CREATE: + if event.mask & pyinotify.IN_CREATE: + # skip directories altogether + if event.mask & pyinotify.IN_ISDIR: + logSys.debug("Ignoring creation of directory %s" % path) + return # check if that is a file we care about if not path in self.__watches: logSys.debug("Ignoring creation of %s we do not monitor" % path) From 9055d925f26a338232a6e017548a68ca9d9d489f Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 25 Jan 2013 14:19:10 -0700 Subject: [PATCH 04/45] Remove unneeded setting of opts.no_network --- fail2ban-testcases | 3 --- 1 file changed, 3 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 20d3b226..99fefd57 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -96,9 +96,6 @@ else: stdout.setFormatter(logging.Formatter(' %(message)s')) logSys.addHandler(stdout) -if opts.no_network is None: - opts.no_network = False - # # Let know the version # From 7fc83196b960e663f073b8e554708c1183a0c34f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 28 Jan 2013 09:46:50 -0500 Subject: [PATCH 05/45] RF: move exceptions used by both client and server into common/exceptions.py this prevents importing of server while operating with client only --- client/beautifier.py | 14 ++++---------- common/exceptions.py | 36 ++++++++++++++++++++++++++++++++++++ server/jails.py | 17 +++-------------- 3 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 common/exceptions.py diff --git a/client/beautifier.py b/client/beautifier.py index a75655e7..7e48016c 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -17,20 +17,14 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# Author: Cyril Jaquier -# -# $Revision$ - -__author__ = "Cyril Jaquier" -__version__ = "$Revision$" -__date__ = "$Date$" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" -from server.jails import UnknownJailException -from server.jails import DuplicateJailException import logging +from common.exceptions import UnknownJailException, DuplicateJailException + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.client.config") diff --git a/common/exceptions.py b/common/exceptions.py new file mode 100644 index 00000000..7e933544 --- /dev/null +++ b/common/exceptions.py @@ -0,0 +1,36 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +"""Fail2Ban exceptions used by both client and server + +""" +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko" +__license__ = "GPL" + +# +# Jails +# +class DuplicateJailException(Exception): + pass + +class UnknownJailException(Exception): + pass + + + diff --git a/server/jails.py b/server/jails.py index 3be38f70..4bf5f971 100644 --- a/server/jails.py +++ b/server/jails.py @@ -17,16 +17,11 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# Author: Cyril Jaquier -# -# $Revision$ - -__author__ = "Cyril Jaquier" -__version__ = "$Revision$" -__date__ = "$Date$" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" +from common.exceptions import DuplicateJailException, UnknownJailException from jail import Jail from threading import Lock @@ -160,9 +155,3 @@ class Jails: finally: self.__lock.release() - -class DuplicateJailException(Exception): - pass - -class UnknownJailException(Exception): - pass From 1eb23cf8afc9481ffcd2f393a291e8c9c6817608 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 28 Jan 2013 09:54:08 -0500 Subject: [PATCH 06/45] BF: do not rely on scripts being under /usr -- might differ eg on Fedora -- rely on import of common.version (Closes gh-112) This is also not ideal, since if there happens to be some systemwide common.version -- we are doomed but otherwise, we cannot keep extending comparison check to /bin, /sbin whatelse --- fail2ban-client | 9 +++++---- fail2ban-regex | 9 +++++---- fail2ban-server | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fail2ban-client b/fail2ban-client index 1d8eb15e..13d018e6 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -27,12 +27,13 @@ import getopt, time, shlex, socket # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -client is also under /usr/ +try: + from common.version import version +except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version -# Now we can import our modules -from common.version import version +# Now we can import the rest of modules from common.protocol import printFormatted from client.csocket import CSocket from client.configurator import Configurator diff --git a/fail2ban-regex b/fail2ban-regex index a42ed96d..f9bc72c1 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -26,13 +26,14 @@ import getopt, sys, time, logging, os # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -regex is also under /usr/ - sys.path.insert(1, "/usr/share/fail2ban") +try: + from common.version import version +except ImportError, e: + sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError -from common.version import version from server.filter import Filter from server.failregex import RegexException diff --git a/fail2ban-server b/fail2ban-server index bd86e6cd..0f3410c9 100755 --- a/fail2ban-server +++ b/fail2ban-server @@ -26,11 +26,12 @@ import getopt, sys, logging, os # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -server is also under /usr/ +try: + from common.version import version +except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version -from common.version import version from server.server import Server # Gets the instance of the logger. From ed386dfe0779067daa4f76f38a0234aeaa50292f Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 15 Mar 2013 14:37:11 -0600 Subject: [PATCH 07/45] Add systemd unit file and tmpfiles.d configuration files --- files/fail2ban-tmpfiles.conf | 1 + files/fail2ban.service | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 files/fail2ban-tmpfiles.conf create mode 100644 files/fail2ban.service diff --git a/files/fail2ban-tmpfiles.conf b/files/fail2ban-tmpfiles.conf new file mode 100644 index 00000000..3fd783f3 --- /dev/null +++ b/files/fail2ban-tmpfiles.conf @@ -0,0 +1 @@ +D /var/run/fail2ban 0755 root root - \ No newline at end of file diff --git a/files/fail2ban.service b/files/fail2ban.service new file mode 100644 index 00000000..35d7fc88 --- /dev/null +++ b/files/fail2ban.service @@ -0,0 +1,12 @@ +[Unit] +Description=Fail2ban Service + +[Service] +Type=forking +ExecStart=/usr/bin/fail2ban-client -x start +ExecStop=/usr/bin/fail2ban-client stop +ExecReload=/usr/bin/fail2ban-client reload +Restart=always + +[Install] +WantedBy=network.target From 32d10e904aef5b5981c6b6abe19bee4690d17c0c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 17 Apr 2013 00:03:36 +1000 Subject: [PATCH 08/45] ENH: more openssh fail messages from openssh source code (CVS 20121205) --- config/filter.d/sshd.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index e4339c78..07b2dd57 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -24,13 +24,16 @@ _daemon = sshd # Values: TEXT # failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from \s*$ + ^%(__prefix_line)sDid not receive identification string from $ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed (?:password|publickey) for .* from (?: port \d*)?(?: ssh\d*)?\s*$ + ^%(__prefix_line)sFailed \S+ for .* from (?: port \d*)?(?: ssh\d*)?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ + ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ ^%(__prefix_line)srefused connect from \S+ \(\)\s*$ + ^%(__prefix_line)sUser .+ from not allowed because a group is listed in DenyGroups\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ # Option: ignoreregex From 6f4dad46f0c2a6505594c6c13ed298ef501decba Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Apr 2013 10:07:01 -0400 Subject: [PATCH 09/45] DOC: slight tune ups to README (we are no longer compatible with python 2.3 ;) ) --- README | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/README b/README index cab683f5..2016e9e1 100644 --- a/README +++ b/README @@ -13,13 +13,13 @@ rules can be defined by the user. Fail2Ban can read multiple log files such as sshd or Apache web server ones. This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs -are available on the project website: http://www.fail2ban.org +are available in fail2ban(1) manpage and on the website http://www.fail2ban.org Installation: ------------- Required: - >=python-2.3 (http://www.python.org) + >=python-2.4 (http://www.python.org) Optional: pyinotify: @@ -38,42 +38,43 @@ To install, just do: This will install Fail2Ban into /usr/share/fail2ban. The executable scripts are placed into /usr/bin. -It is possible that Fail2ban is already packaged for your distribution. In this -case, you should use it. +It is possible that Fail2ban is already packaged for your distribution. In +this case, you should use it. Fail2Ban should be correctly installed now. Just type: > fail2ban-client -h -to see if everything is alright. You should always use fail2ban-client and never -call fail2ban-server directly. +to see if everything is alright. You should always use fail2ban-client and +never call fail2ban-server directly. Configuration: -------------- -You can configure Fail2Ban using the files in /etc/fail2ban. It is -possible to configure the server using commands sent to it by -fail2ban-client. The available commands are described in the -fail2ban-client(1) manpage. Also see fail2ban(1) manpage for further -references and find even more documentation on the website: -http://www.fail2ban.org +You can configure Fail2Ban using the files in /etc/fail2ban. It is possible to +configure the server using commands sent to it by fail2ban-client. The +available commands are described in the fail2ban-client(1) manpage. Also see +fail2ban(1) manpage for further references and find even more documentation on +the website: http://www.fail2ban.org Contact: -------- Website: http://www.fail2ban.org -You need some new features, you found bugs: visit -https://github.com/fail2ban/fail2ban/issues +You need some new features, you found bugs? +visit https://github.com/fail2ban/fail2ban/issues and if your issue is not yet known -- file a bug report. -If you would like to troubleshoot or discuss: join the mailing list +You would like to troubleshoot or discuss? +join the mailing list https://lists.sourceforge.net/lists/listinfo/fail2ban-users -If you just appreciate this program: send kudos to the original author -(Cyril Jaquier: ) or the mailing list +You just appreciate this program: +send kudos to the original author (Cyril Jaquier ) +or better to the mailing list https://lists.sourceforge.net/lists/listinfo/fail2ban-users - +since Fail2Ban is "community-driven" for years now. Thanks: ------- From ddebcab9aae68bb1b93d11bddb63c8c54bec4186 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 17 Apr 2013 09:27:06 -0600 Subject: [PATCH 10/45] Add After, PIDFile, and change WantedBy to multi-user.target in fail2ban.server --- files/fail2ban.service | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/files/fail2ban.service b/files/fail2ban.service index 35d7fc88..9c44042b 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -1,12 +1,14 @@ [Unit] Description=Fail2ban Service +After=syslog.target network.target [Service] Type=forking ExecStart=/usr/bin/fail2ban-client -x start ExecStop=/usr/bin/fail2ban-client stop ExecReload=/usr/bin/fail2ban-client reload +PIDFile=/var/run/fail2ban/fail2ban.pid Restart=always [Install] -WantedBy=network.target +WantedBy=multi-user.target From 76c08cebe9944a297327e30928abdd10e332733d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Apr 2013 11:54:45 -0400 Subject: [PATCH 11/45] DOC: a plugin to thanks for the community support --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 12678936..c571d772 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,10 @@ Nicolas Collignon, Pascal Borreli, blotus: - New features: - Enhancements: +Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom +Hendrikx and other TBN heroes supporting users on fail2ban-users +mailing list and IRC. + ver. 0.8.8 (2012/12/06) - stable ---------- - Fixes: From 41b9f7b6ac4a60d411c58fd5f1305d7539fb104a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 04:38:03 +1000 Subject: [PATCH 12/45] BF: filter.d/sshd "Did not receive identification string" relates to an exploit so document this in sshd-ddos.conf but leave it out of authentication based blocks in sshd.conf --- config/filter.d/sshd-ddos.conf | 7 +++++++ config/filter.d/sshd.conf | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf index 266594ba..58698ced 100644 --- a/config/filter.d/sshd-ddos.conf +++ b/config/filter.d/sshd-ddos.conf @@ -2,6 +2,13 @@ # # Author: Yaroslav Halchenko # +# The regex here also relates to a exploit: +# +# http://www.securityfocus.com/bid/17958/exploit +# The example code here shows the pushing of the exploit straight after +# reading the server version. This is where the client version string normally +# pushed. As such the server will read this unparsible information as +# "Did not receive identification string". [INCLUDES] diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 07b2dd57..b4e645c4 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -24,7 +24,6 @@ _daemon = sshd # Values: TEXT # failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from \s*$ - ^%(__prefix_line)sDid not receive identification string from $ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ ^%(__prefix_line)sFailed \S+ for .* from (?: port \d*)?(?: ssh\d*)?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ From 1331e15ac375426fef89928b5ea5d62a724c6380 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 04:48:51 +1000 Subject: [PATCH 13/45] DOC: guidance for pull requests --- DEVELOP | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DEVELOP b/DEVELOP index a31bf04a..124f4c01 100644 --- a/DEVELOP +++ b/DEVELOP @@ -21,6 +21,18 @@ would like to add to Fail2Ban, the best way to do so it to use the GitHub Pull Request feature. You can find more details on the Fail2Ban wiki (http://www.fail2ban.org/wiki/index.php/Get_Involved) +Pull Requests +============= + +When submitting pull requests on GitHub we ask you to: +* Clearly describe the problem you're solving; +* Don't introduce regressions that will make it hard for systems adminstrators + to update; +* Include test cases (see below); +* Include sample logs (if relevant); +* Include a change to the relevant section of the ChangeLog; and +* Include yourself in THANKS if not already there. + Testing ======= From d1c8b5795233d0a035a027381b68b62913ba57dd Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 04:52:21 +1000 Subject: [PATCH 14/45] DOC: ChangeLog versions and dates for Releasing --- DEVELOP | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEVELOP b/DEVELOP index 124f4c01..438599a4 100644 --- a/DEVELOP +++ b/DEVELOP @@ -269,6 +269,10 @@ Releasing git shortlog -sn 0.8.8.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + Ensure the top of the ChangeLog has the right version and current date. + + Ensure the top entry of the ChangeLog has the right version and current date. + # Update man pages (cd man ; ./generate-man ) From 6b260ab974efd2881cdb555966bafbeb26066263 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 04:53:17 +1000 Subject: [PATCH 15/45] DOC: version/date of release --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index c571d772..f119d310 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.8) 2012/12/06 +Fail2Ban (version 0.8.9) 2013/04/XX ================================================================================ ver. 0.8.9 (2013/04/XXX) - wanna-be-stable From 60fa4b5d7c5748bd3f56e0339e86c7718ece153a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 05:08:45 +1000 Subject: [PATCH 16/45] DOC: begining of ChangeLog --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index f119d310..dce5ed6c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,7 +17,13 @@ Michael Gebetsroither, Orion Poplawski, Artur Penttinen, sebres, Nicolas Collignon, Pascal Borreli, blotus: - Fixes: + Yaroslav Halchenko + * [6f4dad46] Documentation python-2.4 is the minimium version. - New features: + Yaroslav Halchenko + * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} + to provide additional flexibility to system adminstrators. Thanks to + beilber for the idea. Close gh-114. - Enhancements: Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom From dc2f42b24dfc43fb660f2402601aeef21cac9b7e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 06:57:35 +1000 Subject: [PATCH 17/45] DOC: ChangeLog - current HEAD back to ce3ab34 --- ChangeLog | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/ChangeLog b/ChangeLog index dce5ed6c..3e366378 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,12 +19,67 @@ Nicolas Collignon, Pascal Borreli, blotus: - Fixes: Yaroslav Halchenko * [6f4dad46] Documentation python-2.4 is the minimium version. + * [1eb23cf8] do not rely on scripts being under /usr -- might differ eg on + Fedora. Closes gh-112. Thanks to Camusensei for the bug report. + * [bf4d4af1] Changes for atomic writes. Thanks to Steven Hiscocks for + insight. Closes gh-103. + * [ab044b75] delay check for the existence of config directory until read. + * [3b4084d4] fixing up for handling of TAI64N timestamps. + * [154aa38e] do not shutdown logging until all jails stop. + Orion Poplawski + * [e4aedfdc00] pyinotify - use bitwise op on masks and do not try tracking + newly created directories. + Nicolas Collignon + * [39667ff6] Avoid leaking file descriptors. Closes gh-167. + Sergey Brester + * [b6bb2f88 and d17b4153] invalid date recognition, irregular because of + sorting template list. + Steven Hiscocks + * [7a442f07] When changing log target with python2.{4,5} handle KeyError. + Closes gh-147, gh-148. + * [b6a68f51] Fix delaction on server side. Close gh-124. + Daniel Black + * [f0610c01] Allow more that a one word command when changing and Action via + the fail2ban-client. Closes gh-134. - New features: Yaroslav Halchenko * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} to provide additional flexibility to system adminstrators. Thanks to beilber for the idea. Close gh-114. - Enhancements: + Steven Hiscocks + * [c6bd8fc] Add Apache Tomcat date format. Close gh-176. + * [4d80fad] Add Guacmole filter. Close gh-176. + * [3d6791f] Ensure restart of Actions after a check fails occurs + consistently. Closes gh-172. + * [MANY] Improvements to test cases, travis, and code coverage (coveralls). + * [b36835f] Add get cinfo to fail2ban-client. Close gh-124. + * [ce3ab34] Added ability to specify PID file. + Orion Poplawski + * [ddebcab] Enhance fail2ban.service defination dependancies and Pidfile. + Closes gh-142. + Artur Penttinen + * [29d0df5] Add mysqld filter. Closes gh-152. + Erwan Ben Souiden + * [d7d5228] add nagios integration documentation and script to ensure + fail2ban is running. Closes gh-166. + ArndRaphael Brandes + * [bba3fd8] Add Sogo filter. Closes gh-117 + Yaroslav Halchenko + * [MANY] Lots of improvements to log messages and test cases. + * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. + Closes gh-126. Bug report by Michael Heuberger. + * [40c5a2d] adding more of diagnostic messages into -client while starting + the daemon. + Daniel Black + * [3aeb1a9] Add jail.conf manual page. close gh-143. + * [7cd6dab] Added help command to fail2ban-client. close gh-134. + * [c8c7b0b,23bbc60] Better logging of log file read errors. close gh-134. + * [3665e6d] Added code coverage to development process. + Pascal Borreli + * [a2b29b4] Fixed lots of typos in config files and documentation. + Michael Gebetsriother + * [f9b78ba] Add action route to block at routing level. Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom Hendrikx and other TBN heroes supporting users on fail2ban-users From 3e0e0482aef1451c9a991ac1879900333c4e8d4e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 07:07:05 +1000 Subject: [PATCH 18/45] DOC: post release ChangeLog entry --- DEVELOP | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DEVELOP b/DEVELOP index 438599a4..03edebbc 100644 --- a/DEVELOP +++ b/DEVELOP @@ -296,3 +296,13 @@ Releasing # Email users and development list of release TODO notifying distributors etc. + +Post Release: + +Add the following to the top of the ChangeLog + +ver. 0.8.9 (2013/XX/XXX) - wanna-be-stable +- Fixes +- New Features +- Enhancements + From 0a57b6183606d5904109cf8a7009d668acd73e44 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 07:09:07 +1000 Subject: [PATCH 19/45] DOC: developers please rebase and use a single commit --- DEVELOP | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOP b/DEVELOP index 03edebbc..3e8e430a 100644 --- a/DEVELOP +++ b/DEVELOP @@ -28,6 +28,7 @@ When submitting pull requests on GitHub we ask you to: * Clearly describe the problem you're solving; * Don't introduce regressions that will make it hard for systems adminstrators to update; +* If adding a major feature rebase your changes on master and get to a single commit; * Include test cases (see below); * Include sample logs (if relevant); * Include a change to the relevant section of the ChangeLog; and From d4b5e8ec30600d4966e8bfbfb4368b873a403974 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 08:45:20 +1000 Subject: [PATCH 20/45] DOC: credit man page edits --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3e366378..23f46aef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,13 +66,14 @@ Nicolas Collignon, Pascal Borreli, blotus: ArndRaphael Brandes * [bba3fd8] Add Sogo filter. Closes gh-117 Yaroslav Halchenko - * [MANY] Lots of improvements to log messages and test cases. + * [MANY] Lots of improvements to log messages, man pages and test cases. * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. Closes gh-126. Bug report by Michael Heuberger. * [40c5a2d] adding more of diagnostic messages into -client while starting the daemon. Daniel Black * [3aeb1a9] Add jail.conf manual page. close gh-143. + * [MANY] man page edits. * [7cd6dab] Added help command to fail2ban-client. close gh-134. * [c8c7b0b,23bbc60] Better logging of log file read errors. close gh-134. * [3665e6d] Added code coverage to development process. From ed123ea403f735fceeddbc6973958a55d4fec49e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 11:34:44 +1000 Subject: [PATCH 21/45] DOC: tomcat and Guacmole are next release --- ChangeLog | 2 -- 1 file changed, 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23f46aef..fbdf2d48 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,8 +48,6 @@ Nicolas Collignon, Pascal Borreli, blotus: beilber for the idea. Close gh-114. - Enhancements: Steven Hiscocks - * [c6bd8fc] Add Apache Tomcat date format. Close gh-176. - * [4d80fad] Add Guacmole filter. Close gh-176. * [3d6791f] Ensure restart of Actions after a check fails occurs consistently. Closes gh-172. * [MANY] Improvements to test cases, travis, and code coverage (coveralls). From 5413f9b3a18eccc120c547e2f3027205e1463567 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 11:36:37 +1000 Subject: [PATCH 22/45] DOC: move new actions and filters to New Features in ChangeLog --- ChangeLog | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index fbdf2d48..25b1ad20 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,15 @@ Nicolas Collignon, Pascal Borreli, blotus: * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} to provide additional flexibility to system adminstrators. Thanks to beilber for the idea. Close gh-114. + Erwan Ben Souiden + * [d7d5228] add nagios integration documentation and script to ensure + fail2ban is running. Closes gh-166. + Artur Penttinen + * [29d0df5] Add mysqld filter. Closes gh-152. + ArndRaphael Brandes + * [bba3fd8] Add Sogo filter. Closes gh-117 + Michael Gebetsriother + * [f9b78ba] Add action route to block at routing level. - Enhancements: Steven Hiscocks * [3d6791f] Ensure restart of Actions after a check fails occurs @@ -56,13 +65,6 @@ Nicolas Collignon, Pascal Borreli, blotus: Orion Poplawski * [ddebcab] Enhance fail2ban.service defination dependancies and Pidfile. Closes gh-142. - Artur Penttinen - * [29d0df5] Add mysqld filter. Closes gh-152. - Erwan Ben Souiden - * [d7d5228] add nagios integration documentation and script to ensure - fail2ban is running. Closes gh-166. - ArndRaphael Brandes - * [bba3fd8] Add Sogo filter. Closes gh-117 Yaroslav Halchenko * [MANY] Lots of improvements to log messages, man pages and test cases. * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. @@ -77,8 +79,6 @@ Nicolas Collignon, Pascal Borreli, blotus: * [3665e6d] Added code coverage to development process. Pascal Borreli * [a2b29b4] Fixed lots of typos in config files and documentation. - Michael Gebetsriother - * [f9b78ba] Add action route to block at routing level. Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom Hendrikx and other TBN heroes supporting users on fail2ban-users From e5e01187175dafac23d61da640cd2e7639820505 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 18 Apr 2013 12:13:26 +1000 Subject: [PATCH 23/45] DOC: more ChangeLog entries all the way back to 0.8.8 --- ChangeLog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ChangeLog b/ChangeLog index 25b1ad20..eb7e2cfd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,11 +41,14 @@ Nicolas Collignon, Pascal Borreli, blotus: Daniel Black * [f0610c01] Allow more that a one word command when changing and Action via the fail2ban-client. Closes gh-134. + blotus + * [96eb8986] ' and " should also be escaped in action tags Closes gh-109 - New features: Yaroslav Halchenko * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} to provide additional flexibility to system adminstrators. Thanks to beilber for the idea. Close gh-114. + * [3ce53e87] Add exim filter. Erwan Ben Souiden * [d7d5228] add nagios integration documentation and script to ensure fail2ban is running. Closes gh-166. @@ -55,6 +58,12 @@ Nicolas Collignon, Pascal Borreli, blotus: * [bba3fd8] Add Sogo filter. Closes gh-117 Michael Gebetsriother * [f9b78ba] Add action route to block at routing level. + Teodor Micu & Yaroslav Halchenko + * [5f2d383] Add roundcube auth filter. Close Debian bug #699442. + Daniel Black + * [be06b1b] Add action for iptables-ipsets. Close gh-102. + Soulard Morgan + * [f336d9f] Add filter for webmin. Close gh-99 - Enhancements: Steven Hiscocks * [3d6791f] Ensure restart of Actions after a check fails occurs @@ -79,6 +88,8 @@ Nicolas Collignon, Pascal Borreli, blotus: * [3665e6d] Added code coverage to development process. Pascal Borreli * [a2b29b4] Fixed lots of typos in config files and documentation. + hamilton5 + * [7ede1e8] Update dovecot filter config Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom Hendrikx and other TBN heroes supporting users on fail2ban-users From 01499ad0de00b2a0ca2d06959f8643ad0a2f5105 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:07:19 +0100 Subject: [PATCH 24/45] NF: Filters now allow adding of [Init] section similar to actions --- fail2ban/client/actionreader.py | 72 +++++++++----------------- fail2ban/client/configreader.py | 37 +++++++++++++ fail2ban/client/filterreader.py | 45 +++++----------- fail2ban/client/jailreader.py | 47 +++++++++-------- fail2ban/tests/clientreadertestcase.py | 6 +-- 5 files changed, 103 insertions(+), 104 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 787a41c7..b9211a1b 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,66 +28,42 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging -from configreader import ConfigReader +from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class ActionReader(ConfigReader): - - def __init__(self, action, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = action[0] - self.__cInfo = action[1] - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class ActionReader(OptionConfigReader): + + _configOpts = [ + ["string", "actionstart", ""], + ["string", "actionstop", ""], + ["string", "actioncheck", ""], + ["string", "actionban", ""], + ["string", "actionunban", ""], + ] + def read(self): - return ConfigReader.read(self, "action.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "actionstart", ""], - ["string", "actionstop", ""], - ["string", "actioncheck", ""], - ["string", "actionban", ""], - ["string", "actionunban", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) - - if self.has_section("Init"): - for opt in self.options("Init"): - if not self.__cInfo.has_key(opt): - self.__cInfo[opt] = self.get("Init", opt) - + return ConfigReader.read(self, "action.d/" + self._file) + def convert(self): - head = ["set", self.__name] + head = ["set", self._name] stream = list() - stream.append(head + ["addaction", self.__file]) - for opt in self.__opts: + stream.append(head + ["addaction", self._file]) + for opt in self._opts: if opt == "actionstart": - stream.append(head + ["actionstart", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstart", self._file, self._opts[opt]]) elif opt == "actionstop": - stream.append(head + ["actionstop", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstop", self._file, self._opts[opt]]) elif opt == "actioncheck": - stream.append(head + ["actioncheck", self.__file, self.__opts[opt]]) + stream.append(head + ["actioncheck", self._file, self._opts[opt]]) elif opt == "actionban": - stream.append(head + ["actionban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionban", self._file, self._opts[opt]]) elif opt == "actionunban": - stream.append(head + ["actionunban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionunban", self._file, self._opts[opt]]) # cInfo - if self.__cInfo: - for p in self.__cInfo: - stream.append(head + ["setcinfo", self.__file, p, self.__cInfo[p]]) + if self._initOpts: + for p in self._initOpts: + stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]]) return stream - diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6f1e7740..4cb9cd69 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -130,3 +130,40 @@ class ConfigReader(SafeConfigParserWithIncludes): "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values + +class OptionConfigReader(ConfigReader): + + _configOpts = [] + + def __init__(self, file_, jailName, initOpts, **kwargs): + ConfigReader.__init__(self, **kwargs) + self._file = file_ + self._name = jailName + self._initOpts = initOpts + + def setFile(self, fileName): + self._file = fileName + + def getFile(self): + return self.__file + + def setName(self, name): + self._name = name + + def getName(self): + return self._name + + def read(self): + return ConfigReader.read(self, self._file) + + def getOptions(self, pOpts): + self._opts = ConfigReader.getOptions( + self, "Definition", self._configOpts, pOpts) + + if self.has_section("Init"): + for opt in self.options("Init"): + if not self._initOpts.has_key(opt): + self._initOpts[opt] = self.get("Init", opt) + + def convert(self): + raise NotImplementedError diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b00446e..bdfba4d0 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -28,50 +28,33 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging -from configreader import ConfigReader +from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class FilterReader(ConfigReader): - - def __init__(self, fileName, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = fileName - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class FilterReader(OptionConfigReader): + + _configOpts = [ + ["string", "ignoreregex", ""], + ["string", "failregex", ""], + ] + def read(self): - return ConfigReader.read(self, "filter.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "ignoreregex", ""], - ["string", "failregex", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) + return ConfigReader.read(self, "filter.d/" + self._file) def convert(self): stream = list() - for opt in self.__opts: + for opt in self._opts: if opt == "failregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addfailregex", regex]) + stream.append(["set", self._name, "addfailregex", regex]) elif opt == "ignoreregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addignoreregex", regex]) + stream.append(["set", self._name, "addignoreregex", regex]) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6e35bc0b..f6ac09b3 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -38,7 +38,7 @@ logSys = logging.getLogger(__name__) class JailReader(ConfigReader): - actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) @@ -78,8 +78,10 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name, - basedir=self.getBaseDir()) + filterName, filterOpt = JailReader.splitOption( + self.__opts["filter"]) + self.__filter = FilterReader( + filterName, self.__name, filterOpt, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -92,8 +94,9 @@ class JailReader(ConfigReader): try: if not act: # skip empty actions continue - splitAct = JailReader.splitAction(act) - action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir()) + actName, actOpt = JailReader.splitOption(act) + action = ActionReader( + actName, self.__name, actOpt, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -151,23 +154,23 @@ class JailReader(ConfigReader): return stream #@staticmethod - def splitAction(action): - m = JailReader.actionCRE.match(action) + def splitOption(option): + m = JailReader.optionCRE.match(option) d = dict() mgroups = m.groups() if len(mgroups) == 2: - action_name, action_opts = mgroups + option_name, option_opts = mgroups elif len(mgroups) == 1: - action_name, action_opts = mgroups[0], None + option_name, option_opts = mgroups[0], None else: - raise ValueError("While reading action %s we should have got up to " - "2 groups. Got: %r" % (action, mgroups)) - if not action_opts is None: + raise ValueError("While reading option %s we should have got up to " + "2 groups. Got: %r" % (option, mgroups)) + if not option_opts is None: # Huge bad hack :( This method really sucks. TODO Reimplement it. - actions = "" + options = "" escapeChar = None allowComma = False - for c in action_opts: + for c in option_opts: if c in ('"', "'") and not allowComma: # Start escapeChar = c @@ -178,20 +181,20 @@ class JailReader(ConfigReader): allowComma = False else: if c == ',' and allowComma: - actions += "" + options += "" else: - actions += c + options += c # Split using , - actionsSplit = actions.split(',') + optionsSplit = options.split(',') # Replace the tag with , - actionsSplit = [n.replace("", ',') for n in actionsSplit] + optionsSplit = [n.replace("", ',') for n in optionsSplit] - for param in actionsSplit: + for param in optionsSplit: p = param.split('=') try: d[p[0].strip()] = p[1].strip() except IndexError: - logSys.error("Invalid argument %s in '%s'" % (p, action_opts)) - return [action_name, d] - splitAction = staticmethod(splitAction) + logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) + return [option_name, d] + splitOption = staticmethod(splitOption) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d721ef00..02c65d35 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -112,10 +112,10 @@ class JailReaderTest(unittest.TestCase): self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'ssh-iptables') - def testSplitAction(self): + def testSplitOption(self): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitAction(action) + result = JailReader.splitOption(action) self.assertEquals(expected, result) class FilterReaderTest(unittest.TestCase): @@ -140,7 +140,7 @@ class FilterReaderTest(unittest.TestCase): "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"]] - filterReader = FilterReader("testcase01", "testcase01") + filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) From 9672e44d391d155a17627cbe38736ebb4b141045 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:11:41 +0100 Subject: [PATCH 25/45] ENH: Move jail `maxlines` to filter config --- config/filter.d/guacamole.conf | 4 ++++ config/jail.conf | 4 ---- fail2ban/client/filterreader.py | 3 +++ fail2ban/client/jailreader.py | 1 - man/jail.conf.5 | 5 +++++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 272460e3..49cecc5a 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -16,3 +16,7 @@ failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" fa # Values: TEXT # ignoreregex = + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 2 diff --git a/config/jail.conf b/config/jail.conf index 7ed1bbb6..e8d6db05 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -32,9 +32,6 @@ findtime = 600 # "maxretry" is the number of failures before a host get banned. maxretry = 3 -# "maxlines" is number of log lines to buffer for multi-line regex searches -maxlines = 1 - # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling" and "auto". # This option can be overridden in each jail as well. @@ -375,7 +372,6 @@ action = iptables-multiport[name=Guacmole, port="http,https"] sendmail-whois[name=Guacamole, dest=root, sender=fail2ban@example.com] logpath = /var/log/tomcat*/catalina.out maxretry = 5 -maxlines = 2 # Jail for more extended banning of persistent abusers diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index bdfba4d0..09f0e6a8 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -56,5 +56,8 @@ class FilterReader(OptionConfigReader): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self._name, "addignoreregex", regex]) + if self._initOpts: + if 'maxlines' in self._initOpts: + stream.append(["set", self._name, "maxlines", self._initOpts["maxlines"]]) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index f6ac09b3..39acc446 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -65,7 +65,6 @@ class JailReader(ConfigReader): ["string", "logencoding", "auto"], ["string", "backend", "auto"], ["int", "maxretry", 3], - ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 552b0ac0..d571dc7b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -140,6 +140,11 @@ Using Python "string interpolation" mechanisms, other definitions are allowed an baduseragents = IE|wget failregex = useragent=%(baduseragents)s +.PP +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +.TP +\fBmaxlines\fR +specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .PP Filters can also have a section called [INCLUDES]. This is used to read other configuration files. From 5b227b6670e4d25de7f741d6441a1ff12be6b5ae Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:33:42 +0100 Subject: [PATCH 26/45] TST: Add test for FilterReader [Init] `maxlines` override --- fail2ban/tests/clientreadertestcase.py | 12 +++++++++++- fail2ban/tests/files/filter.d/testcase01.conf | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 02c65d35..3e0b555d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -139,7 +139,8 @@ class FilterReaderTest(unittest.TestCase): "error: PAM: )?User not known to the\\nunderlying authentication." "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', - "^.+ john from host 192.168.1.1\\s*$"]] + "^.+ john from host 192.168.1.1\\s*$"], + ['set', 'testcase01', 'maxlines', "1"]] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() @@ -150,6 +151,15 @@ class FilterReaderTest(unittest.TestCase): # is unreliable self.assertEquals(sorted(filterReader.convert()), sorted(output)) + filterReader = FilterReader( + "testcase01", "testcase01", {'maxlines': "5"}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + #filterReader.getOptions(["failregex", "ignoreregex"]) + filterReader.getOptions(None) + output[-1][-1] = "5" + self.assertEquals(sorted(filterReader.convert()), sorted(output)) + class JailsReaderTest(unittest.TestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf index 4a3a95e9..c549572d 100644 --- a/fail2ban/tests/files/filter.d/testcase01.conf +++ b/fail2ban/tests/files/filter.d/testcase01.conf @@ -32,3 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro # Values: TEXT # ignoreregex = ^.+ john from host 192.168.1.1\s*$ + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 1 From b47ea7f81334d8bb03f234653b446bbfe256a859 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 19 Apr 2013 18:17:56 +0100 Subject: [PATCH 27/45] ENH: Remove redundant `maxlines` option from jail reader --- fail2ban/client/jailreader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 39acc446..0599a85e 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -126,8 +126,6 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) - elif opt == "maxlines": - stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. From 4cc3a81cc1cc7ce3fd496e854a4476890a4a6cd4 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:07:23 +0100 Subject: [PATCH 28/45] TST: Move test TZ changes to setUp and tearDown methods --- bin/fail2ban-testcases | 19 +------------------ fail2ban/tests/datedetectortestcase.py | 3 +++ fail2ban/tests/filtertestcase.py | 7 +++++++ fail2ban/tests/utils.py | 19 ++++++++++++++++++- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 68a31786..cd9596a0 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -205,24 +205,7 @@ tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) # testRunner = unittest.TextTestRunner(verbosity=verbosity) -try: - # Set the time to a fixed, known value - # Sun Aug 14 12:00:00 CEST 2005 - # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match - old_TZ = os.environ.get('TZ', None) - os.environ['TZ'] = 'Europe/Zurich' - time.tzset() - MyTime.setTime(1124013600) - - tests_results = testRunner.run(tests) - -finally: # pragma: no cover - # Just for the sake of it reset the TZ - # yoh: move all this into setup/teardown methods within tests - os.environ.pop('TZ') - if old_TZ: - os.environ['TZ'] = old_TZ - time.tzset() +tests_results = testRunner.run(tests) if not tests_results.wasSuccessful(): # pragma: no cover sys.exit(1) diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 23f7a174..534abdbb 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -31,16 +31,19 @@ import unittest from fail2ban.server.datedetector import DateDetector from fail2ban.server.datetemplate import DateTemplate +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.__datedetector = DateDetector() self.__datedetector.addDefaultTemplate() def tearDown(self): """Call after every test case.""" + tearDownMyTime() def testGetEpochTime(self): log = "1138049999 [sshd] error: PAM: Authentication failure" diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fa9340ca..053086d4 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -34,6 +34,7 @@ from fail2ban.server.filterpoll import FilterPoll from fail2ban.server.filter import FileFilter, DNSUtils from fail2ban.server.failmanager import FailManager from fail2ban.server.failmanager import FailManagerEmpty +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -213,6 +214,7 @@ class LogFileMonitor(unittest.TestCase): """ def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') @@ -222,6 +224,7 @@ class LogFileMonitor(unittest.TestCase): self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): + tearDownMyTime() _killfile(self.file, self.name) pass @@ -363,6 +366,7 @@ def get_monitor_failures_testcase(Filter_): count = 0 def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' self.name = '%s-%d' % (testclass_name, self.count) MonitorFailures.count += 1 # so we have unique filenames across tests @@ -380,6 +384,7 @@ def get_monitor_failures_testcase(Filter_): def tearDown(self): + tearDownMyTime() #print "D: SLEEPING A BIT" #import time; time.sleep(5) #print "D: TEARING DOWN" @@ -543,6 +548,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = FileFilter(None) self.filter.setActive(True) # TODO Test this @@ -551,6 +557,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" + tearDownMyTime() diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 6b894193..a5d33983 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,9 +22,11 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback +import logging, os, re, traceback, time from os.path import basename, dirname +from fail2ban.server.mytime import MyTime + # # Following "traceback" functions are adopted from PyMVPA distributed # under MIT/Expat and copyright by PyMVPA developers (i.e. me and @@ -99,3 +101,18 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + +old_TZ = os.environ.get('TZ', None) +def setUpMyTime(): + # Set the time to a fixed, known value + # Sun Aug 14 12:00:00 CEST 2005 + # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match + os.environ['TZ'] = 'Europe/Zurich' + time.tzset() + MyTime.setTime(1124013600) + +def tearDownMyTime(): + os.environ.pop('TZ') + if old_TZ: + os.environ['TZ'] = old_TZ + time.tzset() From 9e684abad7b88e93cd66962798723a72427fc819 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:13:21 +0100 Subject: [PATCH 29/45] TST: Move test gathering to function is test utils --- bin/fail2ban-testcases | 89 +--------------------------------------- fail2ban/tests/utils.py | 90 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 88 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index cd9596a0..15eb085a 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -33,16 +33,8 @@ import unittest, logging, sys, time, os if os.path.exists("fail2ban/__init__.py"): sys.path.insert(0, ".") from fail2ban.version import version -from fail2ban.tests import banmanagertestcase -from fail2ban.tests import clientreadertestcase -from fail2ban.tests import failmanagertestcase -from fail2ban.tests import filtertestcase -from fail2ban.tests import servertestcase -from fail2ban.tests import datedetectortestcase -from fail2ban.tests import actiontestcase -from fail2ban.tests import sockettestcase -from fail2ban.tests.utils import FormatterWithTraceBack +from fail2ban.tests.utils import FormatterWithTraceBack, gatherTests from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option @@ -122,84 +114,7 @@ if not opts.log_level or opts.log_level != 'fatal': # pragma: no cover print "Fail2ban %s test suite. Python %s. Please wait..." \ % (version, str(sys.version).replace('\n', '')) - -# -# Gather the tests -# -if not len(regexps): # pragma: no cover - tests = unittest.TestSuite() -else: # pragma: no cover - import re - class FilteredTestSuite(unittest.TestSuite): - _regexps = [re.compile(r) for r in regexps] - def addTest(self, suite): - suite_str = str(suite) - for r in self._regexps: - if r.search(suite_str): - super(FilteredTestSuite, self).addTest(suite) - return - - tests = FilteredTestSuite() - -# Server -#tests.addTest(unittest.makeSuite(servertestcase.StartStop)) -tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) -tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) -# FailManager -tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) -# BanManager -tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) -# ClientReaders -tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) -# CSocket and AsyncServer -tests.addTest(unittest.makeSuite(sockettestcase.Socket)) - -# Filter -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) - tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) -tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) - -# DateDetector -tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) - -# -# Extensive use-tests of different available filters backends -# - -from fail2ban.server.filterpoll import FilterPoll -filters = [FilterPoll] # always available - -# Additional filters available only if external modules are available -# yoh: Since I do not know better way for parametric tests -# with good old unittest -try: - from fail2ban.server.filtergamin import FilterGamin - filters.append(FilterGamin) -except Exception, e: # pragma: no cover - print "I: Skipping gamin backend testing. Got exception '%s'" % e - -try: - from fail2ban.server.filterpyinotify import FilterPyinotify - filters.append(FilterPyinotify) -except Exception, e: # pragma: no cover - print "I: Skipping pyinotify backend testing. Got exception '%s'" % e - -for Filter_ in filters: - tests.addTest(unittest.makeSuite( - filtertestcase.get_monitor_failures_testcase(Filter_))) - -# Server test for logging elements which break logging used to support -# testcases analysis -tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) - +tests = gatherTests(regexps, opts.no_network) # # Run the tests # diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index a5d33983..57814664 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,11 +22,13 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback, time +import logging, os, re, traceback, time, unittest from os.path import basename, dirname from fail2ban.server.mytime import MyTime +logSys = logging.getLogger(__name__) + # # Following "traceback" functions are adopted from PyMVPA distributed # under MIT/Expat and copyright by PyMVPA developers (i.e. me and @@ -116,3 +118,89 @@ def tearDownMyTime(): if old_TZ: os.environ['TZ'] = old_TZ time.tzset() + +from fail2ban.tests import banmanagertestcase +from fail2ban.tests import clientreadertestcase +from fail2ban.tests import failmanagertestcase +from fail2ban.tests import filtertestcase +from fail2ban.tests import servertestcase +from fail2ban.tests import datedetectortestcase +from fail2ban.tests import actiontestcase +from fail2ban.tests import sockettestcase + +def gatherTests(regexps=None, no_network=False): + if not regexps: # pragma: no cover + tests = unittest.TestSuite() + else: # pragma: no cover + import re + class FilteredTestSuite(unittest.TestSuite): + _regexps = [re.compile(r) for r in regexps] + def addTest(self, suite): + suite_str = str(suite) + for r in self._regexps: + if r.search(suite_str): + super(FilteredTestSuite, self).addTest(suite) + return + + tests = FilteredTestSuite() + + # Server + #tests.addTest(unittest.makeSuite(servertestcase.StartStop)) + tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) + tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) + # FailManager + tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) + # BanManager + tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + # ClientReaders + tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + # CSocket and AsyncServer + tests.addTest(unittest.makeSuite(sockettestcase.Socket)) + + # Filter + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) + tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) + + # DateDetector + tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) + + # + # Extensive use-tests of different available filters backends + # + + from fail2ban.server.filterpoll import FilterPoll + filters = [FilterPoll] # always available + + # Additional filters available only if external modules are available + # yoh: Since I do not know better way for parametric tests + # with good old unittest + try: + from fail2ban.server.filtergamin import FilterGamin + filters.append(FilterGamin) + except Exception, e: # pragma: no cover + logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e) + + try: + from fail2ban.server.filterpyinotify import FilterPyinotify + filters.append(FilterPyinotify) + except Exception, e: # pragma: no cover + logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e) + + for Filter_ in filters: + tests.addTest(unittest.makeSuite( + filtertestcase.get_monitor_failures_testcase(Filter_))) + + # Server test for logging elements which break logging used to support + # testcases analysis + tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) + + return tests From 55810a3c30a745d5b099b40c42d0446b492ea995 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:17:36 +0100 Subject: [PATCH 30/45] TST+RF: Add ability to execute test from setup.py with setuptools Note that the fail2ban version can no longer be imported from "fail2ban.version", as this breaks 2to3 conversion for tests --- .travis.yml | 4 +--- setup.py | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3edadda1..8cfeeff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,8 @@ install: - pip install pyinotify - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi -before_script: - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi diff --git a/setup.py b/setup.py index 57b75225..4b6e304e 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,13 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from distutils.core import setup +try: + import setuptools + from setuptools import setup +except ImportError: + setuptools = None + from distutils.core import setup + try: # python 3.x from distutils.command.build_py import build_py_2to3 as build_py @@ -36,7 +42,23 @@ from os.path import isfile, join, isdir import sys from glob import glob -from fail2ban.version import version +if setuptools and "test" in sys.argv: + import logging + logSys = logging.getLogger("fail2ban") + hdlr = logging.StreamHandler(sys.stdout) + fmt = logging.Formatter("%(asctime)-15s %(message)s") + hdlr.setFormatter(fmt) + logSys.addHandler(hdlr) + if set(["-q", "--quiet"]) & set(sys.argv): + logSys.setLevel(logging.FATAL) + logging.captureWarnings(True) + elif set(["-v", "--verbose"]) & set(sys.argv): + logSys.setLevel(logging.DEBUG) + else: + logSys.setLevel(logging.INFO) +elif "test" in sys.argv: + print("python distribute required to execute fail2ban tests") + print("") longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or @@ -45,9 +67,17 @@ too many password failures. It updates firewall rules to reject the IP address or executes user defined commands.''' +if setuptools: + setup_extra = { + 'test_suite': "fail2ban.tests.utils.gatherTests", + 'use_2to3': True, + } +else: + setup_extra = {} + setup( name = "fail2ban", - version = version, + version = "0.9.0a", description = "Ban IPs that make too many password failures", long_description = longdesc, author = "Cyril Jaquier", @@ -88,7 +118,8 @@ setup( ('/usr/share/doc/fail2ban', ['README', 'DEVELOP', 'doc/run-rootless.txt'] ) - ] + ], + **setup_extra ) # Do some checks after installation From 274227bdfa613bafe026496e554f0167f7cacaf8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 20 Apr 2013 19:40:56 -0400 Subject: [PATCH 31/45] DOC: tune up formatting (spaces) and prelude for the changelog entry --- ChangeLog | 145 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/ChangeLog b/ChangeLog index eb7e2cfd..e60eb12f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,86 +10,89 @@ Fail2Ban (version 0.8.9) 2013/04/XX ver. 0.8.9 (2013/04/XXX) - wanna-be-stable ---------- -This release incorporates 144 (XXX) non-merge commits from 14 -contributors (sorted by number of commits): Yaroslav Halchenko, Daniel -Black, Steven Hiscocks, ArndRa, hamilton5, pigsyn, Erwan Ben Souiden, -Michael Gebetsroither, Orion Poplawski, Artur Penttinen, sebres, -Nicolas Collignon, Pascal Borreli, blotus: +Although primarily a bugfix release, it incorporates many new +enhancements, few new features, but more importantly -- quite extended +tests battery with current 94% coverage. This release incorporates +more than a 100 of non-merge commits from 14 contributors (sorted by +number of commits): Yaroslav Halchenko, Daniel Black, Steven Hiscocks, +ArndRa, hamilton5, pigsyn, Erwan Ben Souiden, Michael Gebetsroither, +Orion Poplawski, Artur Penttinen, sebres, Nicolas Collignon, Pascal +Borreli, blotus: - Fixes: Yaroslav Halchenko - * [6f4dad46] Documentation python-2.4 is the minimium version. - * [1eb23cf8] do not rely on scripts being under /usr -- might differ eg on - Fedora. Closes gh-112. Thanks to Camusensei for the bug report. - * [bf4d4af1] Changes for atomic writes. Thanks to Steven Hiscocks for - insight. Closes gh-103. - * [ab044b75] delay check for the existence of config directory until read. - * [3b4084d4] fixing up for handling of TAI64N timestamps. - * [154aa38e] do not shutdown logging until all jails stop. + * [6f4dad46] Documentation python-2.4 is the minimium version. + * [1eb23cf8] do not rely on scripts being under /usr -- might differ eg on + Fedora. Closes gh-112. Thanks to Camusensei for the bug report. + * [bf4d4af1] Changes for atomic writes. Thanks to Steven Hiscocks for + insight. Closes gh-103. + * [ab044b75] delay check for the existence of config directory until read. + * [3b4084d4] fixing up for handling of TAI64N timestamps. + * [154aa38e] do not shutdown logging until all jails stop. Orion Poplawski - * [e4aedfdc00] pyinotify - use bitwise op on masks and do not try tracking - newly created directories. - Nicolas Collignon - * [39667ff6] Avoid leaking file descriptors. Closes gh-167. - Sergey Brester - * [b6bb2f88 and d17b4153] invalid date recognition, irregular because of - sorting template list. + * [e4aedfdc00] pyinotify - use bitwise op on masks and do not try tracking + newly created directories. + Nicolas Collignon + * [39667ff6] Avoid leaking file descriptors. Closes gh-167. + Sergey Brester + * [b6bb2f88 and d17b4153] invalid date recognition, irregular because of + sorting template list. Steven Hiscocks - * [7a442f07] When changing log target with python2.{4,5} handle KeyError. - Closes gh-147, gh-148. - * [b6a68f51] Fix delaction on server side. Close gh-124. + * [7a442f07] When changing log target with python2.{4,5} handle KeyError. + Closes gh-147, gh-148. + * [b6a68f51] Fix delaction on server side. Closes gh-124. Daniel Black - * [f0610c01] Allow more that a one word command when changing and Action via - the fail2ban-client. Closes gh-134. + * [f0610c01] Allow more that a one word command when changing and Action via + the fail2ban-client. Closes gh-134. blotus - * [96eb8986] ' and " should also be escaped in action tags Closes gh-109 + * [96eb8986] ' and " should also be escaped in action tags Closes gh-109 - New features: Yaroslav Halchenko - * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} - to provide additional flexibility to system adminstrators. Thanks to - beilber for the idea. Close gh-114. - * [3ce53e87] Add exim filter. - Erwan Ben Souiden - * [d7d5228] add nagios integration documentation and script to ensure - fail2ban is running. Closes gh-166. - Artur Penttinen - * [29d0df5] Add mysqld filter. Closes gh-152. - ArndRaphael Brandes - * [bba3fd8] Add Sogo filter. Closes gh-117 - Michael Gebetsriother - * [f9b78ba] Add action route to block at routing level. - Teodor Micu & Yaroslav Halchenko - * [5f2d383] Add roundcube auth filter. Close Debian bug #699442. + * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} + to provide additional flexibility to system adminstrators. Thanks to + beilber for the idea. Closes gh-114. + * [3ce53e87] Add exim filter. + Erwan Ben Souiden + * [d7d5228] add nagios integration documentation and script to ensure + fail2ban is running. Closes gh-166. + Artur Penttinen + * [29d0df5] Add mysqld filter. Closes gh-152. + ArndRaphael Brandes + * [bba3fd8] Add Sogo filter. Closes gh-117. + Michael Gebetsriother + * [f9b78ba] Add action route to block at routing level. + Teodor Micu & Yaroslav Halchenko + * [5f2d383] Add roundcube auth filter. Closes Debian bug #699442. Daniel Black - * [be06b1b] Add action for iptables-ipsets. Close gh-102. - Soulard Morgan - * [f336d9f] Add filter for webmin. Close gh-99 + * [be06b1b] Add action for iptables-ipsets. Closes gh-102. + Soulard Morgan + * [f336d9f] Add filter for webmin. Closes gh-99. - Enhancements: Steven Hiscocks - * [3d6791f] Ensure restart of Actions after a check fails occurs - consistently. Closes gh-172. - * [MANY] Improvements to test cases, travis, and code coverage (coveralls). - * [b36835f] Add get cinfo to fail2ban-client. Close gh-124. - * [ce3ab34] Added ability to specify PID file. - Orion Poplawski - * [ddebcab] Enhance fail2ban.service defination dependancies and Pidfile. - Closes gh-142. + * [3d6791f] Ensure restart of Actions after a check fails occurs + consistently. Closes gh-172. + * [MANY] Improvements to test cases, travis, and code coverage (coveralls). + * [b36835f] Add get cinfo to fail2ban-client. Closes gh-124. + * [ce3ab34] Added ability to specify PID file. + Orion Poplawski + * [ddebcab] Enhance fail2ban.service definition dependencies and Pidfile. + Closes gh-142. Yaroslav Halchenko - * [MANY] Lots of improvements to log messages, man pages and test cases. - * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. - Closes gh-126. Bug report by Michael Heuberger. - * [40c5a2d] adding more of diagnostic messages into -client while starting - the daemon. - Daniel Black - * [3aeb1a9] Add jail.conf manual page. close gh-143. - * [MANY] man page edits. - * [7cd6dab] Added help command to fail2ban-client. close gh-134. - * [c8c7b0b,23bbc60] Better logging of log file read errors. close gh-134. - * [3665e6d] Added code coverage to development process. - Pascal Borreli - * [a2b29b4] Fixed lots of typos in config files and documentation. - hamilton5 - * [7ede1e8] Update dovecot filter config + * [MANY] Lots of improvements to log messages, man pages and test cases. + * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. + Closes gh-126. Bug report by Michael Heuberger. + * [40c5a2d] adding more of diagnostic messages into -client while starting + the daemon. + Daniel Black + * [3aeb1a9] Add jail.conf manual page. Closes gh-143. + * [MANY] man page edits. + * [7cd6dab] Added help command to fail2ban-client. + * [c8c7b0b,23bbc60] Better logging of log file read errors. + * [3665e6d] Added code coverage to development process. + Pascal Borreli + * [a2b29b4] Fixed lots of typos in config files and documentation. + hamilton5 + * [7ede1e8] Update dovecot filter config. Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom Hendrikx and other TBN heroes supporting users on fail2ban-users @@ -100,20 +103,20 @@ ver. 0.8.8 (2012/12/06) - stable - Fixes: Alan Jenkins * [8c38907] Removed 'POSSIBLE BREAK-IN ATTEMPT' from sshd filter to avoid - banning due to misconfigured DNS. Close gh-64 + banning due to misconfigured DNS. Closes gh-64 Yaroslav Halchenko * [83109bc] IMPORTANT: escape the content of (if used in custom action files) since its value could contain arbitrary symbols. Thanks for discovery go to the NBS System security team - * [0935566,5becaf8] Various python 2.4 and 2.5 compatibility fixes. Close gh-83 + * [0935566,5becaf8] Various python 2.4 and 2.5 compatibility fixes. Closes gh-83 * [b159eab] do not enable pyinotify backend if pyinotify < 0.8.3 * [37a2e59] store IP as a base, non-unicode str to avoid spurious messages - in the console. Close gh-91 + in the console. Closes gh-91 - New features: David Engeset * [2d672d1,6288ec2] 'unbanip' command for the client + avoidance of touching - the log file to take 'banip' or 'unbanip' in effect. Close gh-81, gh-86 + the log file to take 'banip' or 'unbanip' in effect. Closes gh-81, gh-86 Yaroslav Halchenko - Enhancements: * [2d66f31] replaced uninformative "Invalid command" message with warning log From f07a92f0f72f17092f7f03128dfaae219a316b52 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 00:58:22 +0100 Subject: [PATCH 32/45] RF: setup.py now imports version number again --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b6e304e..6202fde7 100755 --- a/setup.py +++ b/setup.py @@ -75,9 +75,15 @@ if setuptools: else: setup_extra = {} +# Get version number, avoiding importing fail2ban. +# This is due to tests not functioning for python3 as 2to3 takes place later +f = open("fail2ban/version.py") +exec(f.read()) +f.close() + setup( name = "fail2ban", - version = "0.9.0a", + version = version, description = "Ban IPs that make too many password failures", long_description = longdesc, author = "Cyril Jaquier", From 7341031a30f70fc8f6dacbe6828836a3b262f0e6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 20 Apr 2013 23:12:38 -0400 Subject: [PATCH 33/45] ENH: use os.path.join for consistency -- add "Contributors" to authors --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 6202fde7..f99e4f35 100755 --- a/setup.py +++ b/setup.py @@ -18,8 +18,8 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Steven Hiscocks, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2013 Fail2Ban Contributors" __license__ = "GPL" try: @@ -77,7 +77,7 @@ else: # Get version number, avoiding importing fail2ban. # This is due to tests not functioning for python3 as 2to3 takes place later -f = open("fail2ban/version.py") +f = open(join("fail2ban", "version.py")) exec(f.read()) f.close() @@ -86,7 +86,7 @@ setup( version = version, description = "Ban IPs that make too many password failures", long_description = longdesc, - author = "Cyril Jaquier", + author = "Cyril Jaquier & Fail2Ban Contributors", author_email = "cyril.jaquier@fail2ban.org", url = "http://www.fail2ban.org", license = "GPL", From 12df12f282c63be40dd6475e9d4e2f15bb702f72 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:21:54 +0100 Subject: [PATCH 34/45] BF: Change logging instance logSys `warn` method to `warning` `warn` is long time depreciated method, which may be dropped in python3.4 http://bugs.python.org/issue13235 --- bin/fail2ban-client | 2 +- fail2ban/client/beautifier.py | 2 +- fail2ban/client/configreader.py | 8 ++++---- fail2ban/client/jailreader.py | 2 +- fail2ban/server/actions.py | 4 ++-- fail2ban/server/asyncserver.py | 2 +- fail2ban/server/filter.py | 6 +++--- fail2ban/server/filterpoll.py | 4 ++-- fail2ban/server/transmitter.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 8068d60f..e0b7efc6 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -102,7 +102,7 @@ class Fail2banClient: def __sigTERMhandler(self, signum, frame): # Print a new line because we probably come from wait print - logSys.warn("Caught signal %d. Exiting" % signum) + logSys.warning("Caught signal %d. Exiting" % signum) sys.exit(-1) def __getCmdLineOptions(self, optList): diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 153ab905..a0ff8ff3 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -133,7 +133,7 @@ class Beautifier: c += 1 msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1] except Exception: - logSys.warn("Beautifier error. Please report the error") + logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify " + `response` + " with " + `self.__inputCmd` + " failed") msg = msg + `response` diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6f1e7740..b2d3e392 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -70,7 +70,7 @@ class ConfigReader(SafeConfigParserWithIncludes): # files must carry .conf suffix as well config_files += sorted(glob.glob('%s/*.conf' % config_dir)) else: - logSys.warn("%s exists but not a directory or not accessible" + logSys.warning("%s exists but not a directory or not accessible" % config_dir) # check if files are accessible, warn if any is not accessible @@ -80,7 +80,7 @@ class ConfigReader(SafeConfigParserWithIncludes): if os.access(f, os.R_OK): config_files_accessible.append(f) else: - logSys.warn("%s exists but not accessible - skipping" % f) + logSys.warning("%s exists but not accessible - skipping" % f) if len(config_files_accessible): # at least one config exists and accessible @@ -122,11 +122,11 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] except NoOptionError: if not option[2] == None: - logSys.warn("'%s' not defined in '%s'. Using default one: %r" + logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] except ValueError: - logSys.warn("Wrong value for '" + option[1] + "' in '" + sec + + logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6e35bc0b..d8102971 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -105,7 +105,7 @@ class JailReader(ConfigReader): logSys.debug("Caught exception: %s" % (e,)) return False if not len(self.__actions): - logSys.warn("No actions were defined for %s" % self.__name) + logSys.warning("No actions were defined for %s" % self.__name) return True def convert(self): diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c0373239..b7e58975 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -177,7 +177,7 @@ class Actions(JailThread): aInfo["time"] = bTicket.getTime() aInfo["matches"] = "".join(bTicket.getMatches()) if self.__banManager.addBanTicket(bTicket): - logSys.warn("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionBan(aInfo) return True @@ -217,7 +217,7 @@ class Actions(JailThread): aInfo["failures"] = ticket.getAttempt() aInfo["time"] = ticket.getTime() aInfo["matches"] = "".join(ticket.getMatches()) - logSys.warn("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionUnban(aInfo) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 8c905010..e6af4b26 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -135,7 +135,7 @@ class AsyncServer(asyncore.dispatcher): if os.path.exists(sock): logSys.error("Fail2ban seems to be already running") if force: - logSys.warn("Forcing execution of the server") + logSys.warning("Forcing execution of the server") os.remove(sock) else: raise AsyncServerException("Server already running") diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a7062f16..d4108dc2 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -632,7 +632,7 @@ class FileContainer: try: line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warn("Error decoding line from '%s' with '%s': %s" % + logSys.warning("Error decoding line from '%s' with '%s': %s" % (self.getFileName(), self.getEncoding(), `line`)) if sys.version_info >= (3,): # In python3, must be decoded line = line.decode(self.getEncoding(), 'ignore') @@ -668,11 +668,11 @@ class DNSUtils: try: return socket.gethostbyname_ex(dns)[2] except socket.gaierror: - logSys.warn("Unable to find a corresponding IP address for %s" + logSys.warning("Unable to find a corresponding IP address for %s" % dns) return list() except socket.error, e: - logSys.warn("Socket error raised trying to resolve hostname %s: %s" + logSys.warning("Socket error raised trying to resolve hostname %s: %s" % (dns, e)) return list() dnsToIp = staticmethod(dnsToIp) diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 3217e958..191b95b3 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -131,10 +131,10 @@ class FilterPoll(FileFilter): % (filename, e)) self.__file404Cnt[filename] += 1 if self.__file404Cnt[filename] > 2: - logSys.warn("Too many errors. Setting the jail idle") + logSys.warning("Too many errors. Setting the jail idle") if self.jail: self.jail.setIdle(True) else: - logSys.warn("No jail is assigned to %s" % self) + logSys.warning("No jail is assigned to %s" % self) self.__file404Cnt[filename] = 0 return False diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2d27ff6e..2a4514da 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -55,7 +55,7 @@ class Transmitter: ret = self.__commandHandler(command) ack = 0, ret except Exception, e: - logSys.warn("Command %r has failed. Received %r" + logSys.warning("Command %r has failed. Received %r" % (command, e)) ack = 1, e return ack From c9b1b88bfcde1ed14ca76360b161f102c184895a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:26:30 +0100 Subject: [PATCH 35/45] TST: Ensure files are closed in tests to remove ResourceWarnings --- fail2ban/tests/clientreadertestcase.py | 4 +++- fail2ban/tests/filtertestcase.py | 29 ++++++++++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d721ef00..6fb125e3 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -53,10 +53,12 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - open("%s/%s" % (self.d, fname), "w").write(""" + f = open("%s/%s" % (self.d, fname), "w") + f.write(""" [section] option = %s """ % value) + f.close() def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 053086d4..e540e41e 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -123,7 +123,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, found, output, count) -def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): """Copy lines from one file to another (which might be already open) Returns open fout @@ -132,8 +132,10 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # on old Python st_mtime is int, so we should give at least 1 sec so # polling filter could detect the change time.sleep(1) - if isinstance(fin, str): # pragma: no branch - only used with str in test cases - fin = open(fin, 'r') + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + fin = open(in_, 'r') + else: + fin = in_ # Skip for i in xrange(skip): _ = fin.readline() @@ -151,6 +153,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line fout = open(fout, mode) fout.write('\n'.join(lines)) fout.flush() + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + # Opened earlier, therefore must close it + fin.close() # to give other threads possibly some time to crunch time.sleep(0.1) return fout @@ -291,7 +296,7 @@ class LogFileMonitor(unittest.TestCase): # # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -308,6 +313,7 @@ class LogFileMonitor(unittest.TestCase): def testNewChangeViaGetFailures_move(self): # # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') self.filter.getFailures(self.name) @@ -316,7 +322,7 @@ class LogFileMonitor(unittest.TestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -454,7 +460,7 @@ def get_monitor_failures_testcase(Filter_): def test_rewrite_file(self): # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # What if file gets overridden @@ -468,6 +474,7 @@ def get_monitor_failures_testcase(Filter_): def test_move_file(self): # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time @@ -477,25 +484,25 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) def test_new_bogus_file(self): # to make sure that watching whole directory does not effect - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # create a bogus file in the same directory and see if that doesn't affect - open(self.name + '.bak2', 'w').write('') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + open(self.name + '.bak2', 'w').close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') From 393679341385d24db09a489d06fde494480d7b3c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:28:34 +0100 Subject: [PATCH 36/45] TST: Change depreciated unittest assertEquals method to assertEqual --- fail2ban/tests/clientreadertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 6fb125e3..754f7998 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -118,7 +118,7 @@ class JailReaderTest(unittest.TestCase): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] result = JailReader.splitAction(action) - self.assertEquals(expected, result) + self.assertEqual(expected, result) class FilterReaderTest(unittest.TestCase): @@ -150,7 +150,7 @@ class FilterReaderTest(unittest.TestCase): # Add sort as configreader uses dictionary and therefore order # is unreliable - self.assertEquals(sorted(filterReader.convert()), sorted(output)) + self.assertEqual(sorted(filterReader.convert()), sorted(output)) class JailsReaderTest(unittest.TestCase): From b182c5b5d455e68c343611f095f5caa05c64052a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:30:13 +0100 Subject: [PATCH 37/45] ENH: For python3.2+ use ConfigPaser which replaces SafeConfigParser Current SafeConfigParser alias to be dropped in future python versions --- fail2ban/client/configparserinc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 0ffa0728..7b054876 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -27,8 +27,12 @@ __date__ = '$Date$' __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' -import logging, os -from ConfigParser import SafeConfigParser +import logging, os, sys +if sys.version_info >= (3,2): # pragma: no cover + # SafeConfigParser deprecitated from python 3.2 (renamed ConfigParser) + from configparser import ConfigParser as SafeConfigParser +else: # pragma: no cover + from ConfigParser import SafeConfigParser # Gets the instance of the logger. logSys = logging.getLogger(__name__) From 9d2d907fc198dffee1366bb812d184c685022916 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:59:14 +0100 Subject: [PATCH 38/45] BF: Remove warnings handler which breaks setup.py python2<2.7 and python3<3.2 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f99e4f35..5f65ff37 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ if setuptools and "test" in sys.argv: logSys.addHandler(hdlr) if set(["-q", "--quiet"]) & set(sys.argv): logSys.setLevel(logging.FATAL) - logging.captureWarnings(True) elif set(["-v", "--verbose"]) & set(sys.argv): logSys.setLevel(logging.DEBUG) else: From c95b87c13ceda7e08f0152982590eb93562f6861 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:21:06 +0100 Subject: [PATCH 39/45] ENH: Use os.path.join for filter/action config readers --- fail2ban/client/actionreader.py | 4 ++-- fail2ban/client/filterreader.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index b9211a1b..70c5b3a7 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +import logging, os from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. @@ -44,7 +44,7 @@ class ActionReader(OptionConfigReader): ] def read(self): - return ConfigReader.read(self, "action.d/" + self._file) + return ConfigReader.read(self, os.path.join("action.d", self._file)) def convert(self): head = ["set", self._name] diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 09f0e6a8..19a3f0f3 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +import logging, os from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. @@ -41,7 +41,7 @@ class FilterReader(OptionConfigReader): ] def read(self): - return ConfigReader.read(self, "filter.d/" + self._file) + return ConfigReader.read(self, os.path.join("filter.d", self._file)) def convert(self): stream = list() From 1a43a0bce147ecb6bea6902c633ac1582374ab12 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:22:54 +0100 Subject: [PATCH 40/45] ENH: Rename splitAction to extractOptions in jailreader --- fail2ban/client/jailreader.py | 8 ++++---- fail2ban/tests/clientreadertestcase.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 0599a85e..4a33fdc9 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -77,7 +77,7 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - filterName, filterOpt = JailReader.splitOption( + filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( filterName, self.__name, filterOpt, basedir=self.getBaseDir()) @@ -93,7 +93,7 @@ class JailReader(ConfigReader): try: if not act: # skip empty actions continue - actName, actOpt = JailReader.splitOption(act) + actName, actOpt = JailReader.extractOptions(act) action = ActionReader( actName, self.__name, actOpt, basedir=self.getBaseDir()) ret = action.read() @@ -151,7 +151,7 @@ class JailReader(ConfigReader): return stream #@staticmethod - def splitOption(option): + def extractOptions(option): m = JailReader.optionCRE.match(option) d = dict() mgroups = m.groups() @@ -194,4 +194,4 @@ class JailReader(ConfigReader): except IndexError: logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) return [option_name, d] - splitOption = staticmethod(splitOption) + extractOptions = staticmethod(extractOptions) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 3e0b555d..931a15bc 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -115,7 +115,7 @@ class JailReaderTest(unittest.TestCase): def testSplitOption(self): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitOption(action) + result = JailReader.extractOptions(action) self.assertEquals(expected, result) class FilterReaderTest(unittest.TestCase): From e57505e0740aa1d3f49ba8b9c9b52a0bb45d74bc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:27:32 +0100 Subject: [PATCH 41/45] ENH: Renamed OptionConfigReader to DefinitionInitConfigReader --- fail2ban/client/actionreader.py | 4 ++-- fail2ban/client/configreader.py | 9 ++++++++- fail2ban/client/filterreader.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 70c5b3a7..ca4080d0 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,12 +28,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging, os -from configreader import ConfigReader, OptionConfigReader +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class ActionReader(OptionConfigReader): +class ActionReader(DefinitionInitConfigReader): _configOpts = [ ["string", "actionstart", ""], diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 4cb9cd69..45ea1361 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -131,7 +131,14 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] return values -class OptionConfigReader(ConfigReader): +class DefinitionInitConfigReader(ConfigReader): + """Config reader for files with options grouped in [Definition] and + [Init] sections. + + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 19a3f0f3..62424ddb 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -28,12 +28,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging, os -from configreader import ConfigReader, OptionConfigReader +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class FilterReader(OptionConfigReader): +class FilterReader(DefinitionInitConfigReader): _configOpts = [ ["string", "ignoreregex", ""], From 6f3c66f466a9f9e5abc692d1c0c6131909d94cf3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 13:23:08 +0100 Subject: [PATCH 42/45] ENH: Reimplement warning suppression of setup.py test --quiet --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f65ff37..058c4b21 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ except ImportError: from distutils.command.build_py import build_py from distutils.command.build_scripts import build_scripts from os.path import isfile, join, isdir -import sys +import sys, warnings from glob import glob if setuptools and "test" in sys.argv: @@ -51,6 +51,8 @@ if setuptools and "test" in sys.argv: logSys.addHandler(hdlr) if set(["-q", "--quiet"]) & set(sys.argv): logSys.setLevel(logging.FATAL) + warnings.simplefilter("ignore") + sys.warnoptions.append("ignore") elif set(["-v", "--verbose"]) & set(sys.argv): logSys.setLevel(logging.DEBUG) else: From dadd6aed2f9d20a6d5dc08264433e9b4344ebbe9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 17:39:56 +0100 Subject: [PATCH 43/45] BF+TST: Correctly reset time in tearDownMyTime --- fail2ban/tests/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 57814664..95646d63 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -118,6 +118,7 @@ def tearDownMyTime(): if old_TZ: os.environ['TZ'] = old_TZ time.tzset() + MyTime.myTime = None from fail2ban.tests import banmanagertestcase from fail2ban.tests import clientreadertestcase From 1fcb5efbd79781da5555235a8512e74e2cfa8ca7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 00:01:30 -0400 Subject: [PATCH 44/45] ENH: make fail2ban-regex aware of possible maxlines in the filter config file --- bin/fail2ban-regex | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 9233a671..e3d75f20 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -70,6 +70,7 @@ class Fail2banRegex: self.__ignoreregex = list() self.__failregex = list() self.__verbose = False + self.__maxlines_set = False # so we allow to override maxlines in cmdline self.encoding = locale.getpreferredencoding() # Setup logging logging.getLogger("fail2ban").handlers = [] @@ -126,6 +127,11 @@ class Fail2banRegex: print "Report bugs to https://github.com/fail2ban/fail2ban/issues" dispUsage = staticmethod(dispUsage) + def setMaxLines(self, v): + if not self.__maxlines_set: + self.__filter.setMaxLines(int(v)) + self.__maxlines_set = True + def getCmdLineOptions(self, optList): """ Gets the command line options """ @@ -142,7 +148,7 @@ class Fail2banRegex: self.encoding = opt[1] elif opt[0] in ["-l", "--maxlines"]: try: - self.__filter.setMaxLines(int(opt[1])) + self.setMaxLines(opt[1]) except ValueError: print "Invlaid value for maxlines: %s" % ( opt[1]) @@ -203,6 +209,20 @@ class Fail2banRegex: print "No section headers in " + value print return False + + # Read out and set possible value of maxlines + try: + maxlines = reader.get("Init", "maxlines") + except NoSectionError, NoOptionError: + # No [Init].maxlines found. + pass + else: + try: + self.setMaxLines(maxlines) + except ValueError: + print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() + return False else: if len(value) > 53: stripReg = value[0:50] + "..." @@ -210,6 +230,8 @@ class Fail2banRegex: stripReg = value print "Use regex line : " + stripReg self.__failregex = [RegexStat(value)] + + print "Use maxlines : %d" % self.__filter.getMaxLines() return True def testIgnoreRegex(self, line): From 54bae189a355d904a7eab498a01b2b829f8ee68e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 10:10:20 -0400 Subject: [PATCH 45/45] Beef up changelog for 0.9 --- ChangeLog | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index baa854a9..e11bbde8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,15 +8,27 @@ Fail2Ban (version 0.9.0a) 20??/??/?? ================================================================================ -ver. 0.9.0 (20??/??/??) - alpha +ver. 0.9.0 (2013/04/??) - alpha ---------- -Will carry all fixes in 0.8.x series and new features and enhancements +Carries all fixes in 0.8.9 and new features and enhancements. Nearly +all development is thanks to Steven Hiscocks (THANKS!) with only +code-review and minor additions from Yaroslav Halchenko. -- Fixes: +- Refactoring: + Steven Hiscocks + * [..5aef036] Core functionality moved into fail2ban/ module. + Closes gh-26 - New features: Steven Hiscocks - * Multiline failregex. Close gh-54 + * [..c7ae460] Multiline failregex. Close gh-54 + * [8af32ed] Guacamole filter and support for Apache Tomcat date + format + * [..4869186] Python3 support +- Enhancements + Steven Hiscocks + * Replacing use of deprecated API (.warning, .assertEqual, etc) + * [..a648cc2] Filters can have options now too ver. 0.8.9 (2013/04/XXX) - wanna-be-stable ----------