diff --git a/.travis.yml b/.travis.yml index 3edadda1..8cfeeff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,8 @@ install: - pip install pyinotify - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi -before_script: - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi diff --git a/ChangeLog b/ChangeLog index 9253dfc5..e11bbde8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,48 +8,138 @@ Fail2Ban (version 0.9.0a) 20??/??/?? ================================================================================ -ver. 0.9.0 (20??/??/??) - alpha +ver. 0.9.0 (2013/04/??) - alpha ---------- -Will carry all fixes in 0.8.x series and new features and enhancements +Carries all fixes in 0.8.9 and new features and enhancements. Nearly +all development is thanks to Steven Hiscocks (THANKS!) with only +code-review and minor additions from Yaroslav Halchenko. -- Fixes: +- Refactoring: + Steven Hiscocks + * [..5aef036] Core functionality moved into fail2ban/ module. + Closes gh-26 - New features: Steven Hiscocks - * Multiline failregex. Close gh-54 + * [..c7ae460] Multiline failregex. Close gh-54 + * [8af32ed] Guacamole filter and support for Apache Tomcat date + format + * [..4869186] Python3 support +- Enhancements + Steven Hiscocks + * Replacing use of deprecated API (.warning, .assertEqual, etc) + * [..a648cc2] Filters can have options now too ver. 0.8.9 (2013/04/XXX) - wanna-be-stable ---------- -This release incorporates 144 (XXX) non-merge commits from 14 -contributors (sorted by number of commits): Yaroslav Halchenko, Daniel -Black, Steven Hiscocks, ArndRa, hamilton5, pigsyn, Erwan Ben Souiden, -Michael Gebetsroither, Orion Poplawski, Artur Penttinen, sebres, -Nicolas Collignon, Pascal Borreli, blotus: +Although primarily a bugfix release, it incorporates many new +enhancements, few new features, but more importantly -- quite extended +tests battery with current 94% coverage. This release incorporates +more than a 100 of non-merge commits from 14 contributors (sorted by +number of commits): Yaroslav Halchenko, Daniel Black, Steven Hiscocks, +ArndRa, hamilton5, pigsyn, Erwan Ben Souiden, Michael Gebetsroither, +Orion Poplawski, Artur Penttinen, sebres, Nicolas Collignon, Pascal +Borreli, blotus: - Fixes: + Yaroslav Halchenko + * [6f4dad46] Documentation python-2.4 is the minimium version. + * [1eb23cf8] do not rely on scripts being under /usr -- might differ eg on + Fedora. Closes gh-112. Thanks to Camusensei for the bug report. + * [bf4d4af1] Changes for atomic writes. Thanks to Steven Hiscocks for + insight. Closes gh-103. + * [ab044b75] delay check for the existence of config directory until read. + * [3b4084d4] fixing up for handling of TAI64N timestamps. + * [154aa38e] do not shutdown logging until all jails stop. + Orion Poplawski + * [e4aedfdc00] pyinotify - use bitwise op on masks and do not try tracking + newly created directories. + Nicolas Collignon + * [39667ff6] Avoid leaking file descriptors. Closes gh-167. + Sergey Brester + * [b6bb2f88 and d17b4153] invalid date recognition, irregular because of + sorting template list. + Steven Hiscocks + * [7a442f07] When changing log target with python2.{4,5} handle KeyError. + Closes gh-147, gh-148. + * [b6a68f51] Fix delaction on server side. Closes gh-124. + Daniel Black + * [f0610c01] Allow more that a one word command when changing and Action via + the fail2ban-client. Closes gh-134. + blotus + * [96eb8986] ' and " should also be escaped in action tags Closes gh-109 - New features: + Yaroslav Halchenko + * [9ba27353] Add support for jail.d/{confilefile} and fail2ban.d/{configfile} + to provide additional flexibility to system adminstrators. Thanks to + beilber for the idea. Closes gh-114. + * [3ce53e87] Add exim filter. + Erwan Ben Souiden + * [d7d5228] add nagios integration documentation and script to ensure + fail2ban is running. Closes gh-166. + Artur Penttinen + * [29d0df5] Add mysqld filter. Closes gh-152. + ArndRaphael Brandes + * [bba3fd8] Add Sogo filter. Closes gh-117. + Michael Gebetsriother + * [f9b78ba] Add action route to block at routing level. + Teodor Micu & Yaroslav Halchenko + * [5f2d383] Add roundcube auth filter. Closes Debian bug #699442. + Daniel Black + * [be06b1b] Add action for iptables-ipsets. Closes gh-102. + Soulard Morgan + * [f336d9f] Add filter for webmin. Closes gh-99. - Enhancements: + Steven Hiscocks + * [3d6791f] Ensure restart of Actions after a check fails occurs + consistently. Closes gh-172. + * [MANY] Improvements to test cases, travis, and code coverage (coveralls). + * [b36835f] Add get cinfo to fail2ban-client. Closes gh-124. + * [ce3ab34] Added ability to specify PID file. + Orion Poplawski + * [ddebcab] Enhance fail2ban.service definition dependencies and Pidfile. + Closes gh-142. + Yaroslav Halchenko + * [MANY] Lots of improvements to log messages, man pages and test cases. + * [91d5736] Postfix filter improvements - empty helo, from and rcpt to. + Closes gh-126. Bug report by Michael Heuberger. + * [40c5a2d] adding more of diagnostic messages into -client while starting + the daemon. + Daniel Black + * [3aeb1a9] Add jail.conf manual page. Closes gh-143. + * [MANY] man page edits. + * [7cd6dab] Added help command to fail2ban-client. + * [c8c7b0b,23bbc60] Better logging of log file read errors. + * [3665e6d] Added code coverage to development process. + Pascal Borreli + * [a2b29b4] Fixed lots of typos in config files and documentation. + hamilton5 + * [7ede1e8] Update dovecot filter config. + +Special Kudos also go to Fabian Wenk, Arturo 'Buanzo' Busleiman, Tom +Hendrikx and other TBN heroes supporting users on fail2ban-users +mailing list and IRC. ver. 0.8.8 (2012/12/06) - stable ---------- - Fixes: Alan Jenkins * [8c38907] Removed 'POSSIBLE BREAK-IN ATTEMPT' from sshd filter to avoid - banning due to misconfigured DNS. Close gh-64 + banning due to misconfigured DNS. Closes gh-64 Yaroslav Halchenko * [83109bc] IMPORTANT: escape the content of (if used in custom action files) since its value could contain arbitrary symbols. Thanks for discovery go to the NBS System security team - * [0935566,5becaf8] Various python 2.4 and 2.5 compatibility fixes. Close gh-83 + * [0935566,5becaf8] Various python 2.4 and 2.5 compatibility fixes. Closes gh-83 * [b159eab] do not enable pyinotify backend if pyinotify < 0.8.3 * [37a2e59] store IP as a base, non-unicode str to avoid spurious messages - in the console. Close gh-91 + in the console. Closes gh-91 - New features: David Engeset * [2d672d1,6288ec2] 'unbanip' command for the client + avoidance of touching - the log file to take 'banip' or 'unbanip' in effect. Close gh-81, gh-86 + the log file to take 'banip' or 'unbanip' in effect. Closes gh-81, gh-86 Yaroslav Halchenko - Enhancements: * [2d66f31] replaced uninformative "Invalid command" message with warning log diff --git a/DEVELOP b/DEVELOP index 34c07adc..9d273d0b 100644 --- a/DEVELOP +++ b/DEVELOP @@ -21,6 +21,19 @@ would like to add to Fail2Ban, the best way to do so it to use the GitHub Pull Request feature. You can find more details on the Fail2Ban wiki (http://www.fail2ban.org/wiki/index.php/Get_Involved) +Pull Requests +============= + +When submitting pull requests on GitHub we ask you to: +* Clearly describe the problem you're solving; +* Don't introduce regressions that will make it hard for systems adminstrators + to update; +* If adding a major feature rebase your changes on master and get to a single commit; +* Include test cases (see below); +* Include sample logs (if relevant); +* Include a change to the relevant section of the ChangeLog; and +* Include yourself in THANKS if not already there. + Testing ======= @@ -257,6 +270,10 @@ Releasing git shortlog -sn 0.8.8.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + Ensure the top of the ChangeLog has the right version and current date. + + Ensure the top entry of the ChangeLog has the right version and current date. + # Update man pages (cd man ; ./generate-man ) @@ -280,3 +297,13 @@ Releasing # Email users and development list of release TODO notifying distributors etc. + +Post Release: + +Add the following to the top of the ChangeLog + +ver. 0.8.9 (2013/XX/XXX) - wanna-be-stable +- Fixes +- New Features +- Enhancements + diff --git a/README b/README index e8c0dcd2..77a56af7 100644 --- a/README +++ b/README @@ -13,13 +13,13 @@ rules can be defined by the user. Fail2Ban can read multiple log files such as sshd or Apache web server ones. This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs -are available on the project website: http://www.fail2ban.org +are available in fail2ban(1) manpage and on the website http://www.fail2ban.org Installation: ------------- Required: - >=python-2.3 or >=python-3.0 (http://www.python.org) + >=python-2.4 or >=python-3.0 (http://www.python.org) Optional: pyinotify: @@ -38,42 +38,43 @@ To install, just do: This will install Fail2Ban into /usr/share/fail2ban. The executable scripts are placed into /usr/bin. -It is possible that Fail2ban is already packaged for your distribution. In this -case, you should use it. +It is possible that Fail2ban is already packaged for your distribution. In +this case, you should use it. Fail2Ban should be correctly installed now. Just type: > fail2ban-client -h -to see if everything is alright. You should always use fail2ban-client and never -call fail2ban-server directly. +to see if everything is alright. You should always use fail2ban-client and +never call fail2ban-server directly. Configuration: -------------- -You can configure Fail2Ban using the files in /etc/fail2ban. It is -possible to configure the server using commands sent to it by -fail2ban-client. The available commands are described in the -fail2ban-client(1) manpage. Also see fail2ban(1) manpage for further -references and find even more documentation on the website: -http://www.fail2ban.org +You can configure Fail2Ban using the files in /etc/fail2ban. It is possible to +configure the server using commands sent to it by fail2ban-client. The +available commands are described in the fail2ban-client(1) manpage. Also see +fail2ban(1) manpage for further references and find even more documentation on +the website: http://www.fail2ban.org Contact: -------- Website: http://www.fail2ban.org -You need some new features, you found bugs: visit -https://github.com/fail2ban/fail2ban/issues +You need some new features, you found bugs? +visit https://github.com/fail2ban/fail2ban/issues and if your issue is not yet known -- file a bug report. -If you would like to troubleshoot or discuss: join the mailing list +You would like to troubleshoot or discuss? +join the mailing list https://lists.sourceforge.net/lists/listinfo/fail2ban-users -If you just appreciate this program: send kudos to the original author -(Cyril Jaquier: ) or the mailing list +You just appreciate this program: +send kudos to the original author (Cyril Jaquier ) +or better to the mailing list https://lists.sourceforge.net/lists/listinfo/fail2ban-users - +since Fail2Ban is "community-driven" for years now. Thanks: ------- diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 8068d60f..e0b7efc6 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -102,7 +102,7 @@ class Fail2banClient: def __sigTERMhandler(self, signum, frame): # Print a new line because we probably come from wait print - logSys.warn("Caught signal %d. Exiting" % signum) + logSys.warning("Caught signal %d. Exiting" % signum) sys.exit(-1) def __getCmdLineOptions(self, optList): diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 9233a671..e3d75f20 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -70,6 +70,7 @@ class Fail2banRegex: self.__ignoreregex = list() self.__failregex = list() self.__verbose = False + self.__maxlines_set = False # so we allow to override maxlines in cmdline self.encoding = locale.getpreferredencoding() # Setup logging logging.getLogger("fail2ban").handlers = [] @@ -126,6 +127,11 @@ class Fail2banRegex: print "Report bugs to https://github.com/fail2ban/fail2ban/issues" dispUsage = staticmethod(dispUsage) + def setMaxLines(self, v): + if not self.__maxlines_set: + self.__filter.setMaxLines(int(v)) + self.__maxlines_set = True + def getCmdLineOptions(self, optList): """ Gets the command line options """ @@ -142,7 +148,7 @@ class Fail2banRegex: self.encoding = opt[1] elif opt[0] in ["-l", "--maxlines"]: try: - self.__filter.setMaxLines(int(opt[1])) + self.setMaxLines(opt[1]) except ValueError: print "Invlaid value for maxlines: %s" % ( opt[1]) @@ -203,6 +209,20 @@ class Fail2banRegex: print "No section headers in " + value print return False + + # Read out and set possible value of maxlines + try: + maxlines = reader.get("Init", "maxlines") + except NoSectionError, NoOptionError: + # No [Init].maxlines found. + pass + else: + try: + self.setMaxLines(maxlines) + except ValueError: + print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() + return False else: if len(value) > 53: stripReg = value[0:50] + "..." @@ -210,6 +230,8 @@ class Fail2banRegex: stripReg = value print "Use regex line : " + stripReg self.__failregex = [RegexStat(value)] + + print "Use maxlines : %d" % self.__filter.getMaxLines() return True def testIgnoreRegex(self, line): diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 68a31786..15eb085a 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -33,16 +33,8 @@ import unittest, logging, sys, time, os if os.path.exists("fail2ban/__init__.py"): sys.path.insert(0, ".") from fail2ban.version import version -from fail2ban.tests import banmanagertestcase -from fail2ban.tests import clientreadertestcase -from fail2ban.tests import failmanagertestcase -from fail2ban.tests import filtertestcase -from fail2ban.tests import servertestcase -from fail2ban.tests import datedetectortestcase -from fail2ban.tests import actiontestcase -from fail2ban.tests import sockettestcase -from fail2ban.tests.utils import FormatterWithTraceBack +from fail2ban.tests.utils import FormatterWithTraceBack, gatherTests from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option @@ -122,107 +114,13 @@ if not opts.log_level or opts.log_level != 'fatal': # pragma: no cover print "Fail2ban %s test suite. Python %s. Please wait..." \ % (version, str(sys.version).replace('\n', '')) - -# -# Gather the tests -# -if not len(regexps): # pragma: no cover - tests = unittest.TestSuite() -else: # pragma: no cover - import re - class FilteredTestSuite(unittest.TestSuite): - _regexps = [re.compile(r) for r in regexps] - def addTest(self, suite): - suite_str = str(suite) - for r in self._regexps: - if r.search(suite_str): - super(FilteredTestSuite, self).addTest(suite) - return - - tests = FilteredTestSuite() - -# Server -#tests.addTest(unittest.makeSuite(servertestcase.StartStop)) -tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) -tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) -# FailManager -tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) -# BanManager -tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) -# ClientReaders -tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) -# CSocket and AsyncServer -tests.addTest(unittest.makeSuite(sockettestcase.Socket)) - -# Filter -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) - tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) -tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) - -# DateDetector -tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) - -# -# Extensive use-tests of different available filters backends -# - -from fail2ban.server.filterpoll import FilterPoll -filters = [FilterPoll] # always available - -# Additional filters available only if external modules are available -# yoh: Since I do not know better way for parametric tests -# with good old unittest -try: - from fail2ban.server.filtergamin import FilterGamin - filters.append(FilterGamin) -except Exception, e: # pragma: no cover - print "I: Skipping gamin backend testing. Got exception '%s'" % e - -try: - from fail2ban.server.filterpyinotify import FilterPyinotify - filters.append(FilterPyinotify) -except Exception, e: # pragma: no cover - print "I: Skipping pyinotify backend testing. Got exception '%s'" % e - -for Filter_ in filters: - tests.addTest(unittest.makeSuite( - filtertestcase.get_monitor_failures_testcase(Filter_))) - -# Server test for logging elements which break logging used to support -# testcases analysis -tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) - +tests = gatherTests(regexps, opts.no_network) # # Run the tests # testRunner = unittest.TextTestRunner(verbosity=verbosity) -try: - # Set the time to a fixed, known value - # Sun Aug 14 12:00:00 CEST 2005 - # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match - old_TZ = os.environ.get('TZ', None) - os.environ['TZ'] = 'Europe/Zurich' - time.tzset() - MyTime.setTime(1124013600) - - tests_results = testRunner.run(tests) - -finally: # pragma: no cover - # Just for the sake of it reset the TZ - # yoh: move all this into setup/teardown methods within tests - os.environ.pop('TZ') - if old_TZ: - os.environ['TZ'] = old_TZ - time.tzset() +tests_results = testRunner.run(tests) if not tests_results.wasSuccessful(): # pragma: no cover sys.exit(1) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 272460e3..49cecc5a 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -16,3 +16,7 @@ failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" fa # Values: TEXT # ignoreregex = + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 2 diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf index 266594ba..58698ced 100644 --- a/config/filter.d/sshd-ddos.conf +++ b/config/filter.d/sshd-ddos.conf @@ -2,6 +2,13 @@ # # Author: Yaroslav Halchenko # +# The regex here also relates to a exploit: +# +# http://www.securityfocus.com/bid/17958/exploit +# The example code here shows the pushing of the exploit straight after +# reading the server version. This is where the client version string normally +# pushed. As such the server will read this unparsible information as +# "Did not receive identification string". [INCLUDES] diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index e4339c78..b4e645c4 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -25,12 +25,14 @@ _daemon = sshd # failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from \s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed (?:password|publickey) for .* from (?: port \d*)?(?: ssh\d*)?\s*$ + ^%(__prefix_line)sFailed \S+ for .* from (?: port \d*)?(?: ssh\d*)?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ + ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ ^%(__prefix_line)srefused connect from \S+ \(\)\s*$ + ^%(__prefix_line)sUser .+ from not allowed because a group is listed in DenyGroups\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ # Option: ignoreregex diff --git a/config/jail.conf b/config/jail.conf index 337fe0b0..651ac3b3 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -41,9 +41,6 @@ findtime = 600 # "maxretry" is the number of failures before a host get banned. maxretry = 5 -# "maxlines" is number of log lines to buffer for multi-line regex searches -maxlines = 1 - # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling" and "auto". # This option can be overridden in each jail as well. @@ -535,8 +532,6 @@ enabled = false filter = guacamole port = http,https logpath = /var/log/tomcat*/catalina.out -maxlines = 2 - # Jail for more extended banning of persistent abusers # !!! WARNING !!! diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 787a41c7..ca4080d0 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -27,67 +27,43 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging -from configreader import ConfigReader +import logging, os +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class ActionReader(ConfigReader): - - def __init__(self, action, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = action[0] - self.__cInfo = action[1] - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class ActionReader(DefinitionInitConfigReader): + + _configOpts = [ + ["string", "actionstart", ""], + ["string", "actionstop", ""], + ["string", "actioncheck", ""], + ["string", "actionban", ""], + ["string", "actionunban", ""], + ] + def read(self): - return ConfigReader.read(self, "action.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "actionstart", ""], - ["string", "actionstop", ""], - ["string", "actioncheck", ""], - ["string", "actionban", ""], - ["string", "actionunban", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) - - if self.has_section("Init"): - for opt in self.options("Init"): - if not self.__cInfo.has_key(opt): - self.__cInfo[opt] = self.get("Init", opt) - + return ConfigReader.read(self, os.path.join("action.d", self._file)) + def convert(self): - head = ["set", self.__name] + head = ["set", self._name] stream = list() - stream.append(head + ["addaction", self.__file]) - for opt in self.__opts: + stream.append(head + ["addaction", self._file]) + for opt in self._opts: if opt == "actionstart": - stream.append(head + ["actionstart", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstart", self._file, self._opts[opt]]) elif opt == "actionstop": - stream.append(head + ["actionstop", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstop", self._file, self._opts[opt]]) elif opt == "actioncheck": - stream.append(head + ["actioncheck", self.__file, self.__opts[opt]]) + stream.append(head + ["actioncheck", self._file, self._opts[opt]]) elif opt == "actionban": - stream.append(head + ["actionban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionban", self._file, self._opts[opt]]) elif opt == "actionunban": - stream.append(head + ["actionunban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionunban", self._file, self._opts[opt]]) # cInfo - if self.__cInfo: - for p in self.__cInfo: - stream.append(head + ["setcinfo", self.__file, p, self.__cInfo[p]]) + if self._initOpts: + for p in self._initOpts: + stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]]) return stream - diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 153ab905..a0ff8ff3 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -133,7 +133,7 @@ class Beautifier: c += 1 msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1] except Exception: - logSys.warn("Beautifier error. Please report the error") + logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify " + `response` + " with " + `self.__inputCmd` + " failed") msg = msg + `response` diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 0ffa0728..7b054876 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -27,8 +27,12 @@ __date__ = '$Date$' __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' -import logging, os -from ConfigParser import SafeConfigParser +import logging, os, sys +if sys.version_info >= (3,2): # pragma: no cover + # SafeConfigParser deprecitated from python 3.2 (renamed ConfigParser) + from configparser import ConfigParser as SafeConfigParser +else: # pragma: no cover + from ConfigParser import SafeConfigParser # Gets the instance of the logger. logSys = logging.getLogger(__name__) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6f1e7740..9a1fedaf 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -70,7 +70,7 @@ class ConfigReader(SafeConfigParserWithIncludes): # files must carry .conf suffix as well config_files += sorted(glob.glob('%s/*.conf' % config_dir)) else: - logSys.warn("%s exists but not a directory or not accessible" + logSys.warning("%s exists but not a directory or not accessible" % config_dir) # check if files are accessible, warn if any is not accessible @@ -80,7 +80,7 @@ class ConfigReader(SafeConfigParserWithIncludes): if os.access(f, os.R_OK): config_files_accessible.append(f) else: - logSys.warn("%s exists but not accessible - skipping" % f) + logSys.warning("%s exists but not accessible - skipping" % f) if len(config_files_accessible): # at least one config exists and accessible @@ -122,11 +122,55 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] except NoOptionError: if not option[2] == None: - logSys.warn("'%s' not defined in '%s'. Using default one: %r" + logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] except ValueError: - logSys.warn("Wrong value for '" + option[1] + "' in '" + sec + + logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values + +class DefinitionInitConfigReader(ConfigReader): + """Config reader for files with options grouped in [Definition] and + [Init] sections. + + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ + + _configOpts = [] + + def __init__(self, file_, jailName, initOpts, **kwargs): + ConfigReader.__init__(self, **kwargs) + self._file = file_ + self._name = jailName + self._initOpts = initOpts + + def setFile(self, fileName): + self._file = fileName + + def getFile(self): + return self.__file + + def setName(self, name): + self._name = name + + def getName(self): + return self._name + + def read(self): + return ConfigReader.read(self, self._file) + + def getOptions(self, pOpts): + self._opts = ConfigReader.getOptions( + self, "Definition", self._configOpts, pOpts) + + if self.has_section("Init"): + for opt in self.options("Init"): + if not self._initOpts.has_key(opt): + self._initOpts[opt] = self.get("Init", opt) + + def convert(self): + raise NotImplementedError diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b00446e..62424ddb 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -27,51 +27,37 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging -from configreader import ConfigReader +import logging, os +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class FilterReader(ConfigReader): - - def __init__(self, fileName, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = fileName - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class FilterReader(DefinitionInitConfigReader): + + _configOpts = [ + ["string", "ignoreregex", ""], + ["string", "failregex", ""], + ] + def read(self): - return ConfigReader.read(self, "filter.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "ignoreregex", ""], - ["string", "failregex", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) + return ConfigReader.read(self, os.path.join("filter.d", self._file)) def convert(self): stream = list() - for opt in self.__opts: + for opt in self._opts: if opt == "failregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addfailregex", regex]) + stream.append(["set", self._name, "addfailregex", regex]) elif opt == "ignoreregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addignoreregex", regex]) + stream.append(["set", self._name, "addignoreregex", regex]) + if self._initOpts: + if 'maxlines' in self._initOpts: + stream.append(["set", self._name, "maxlines", self._initOpts["maxlines"]]) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6e35bc0b..13432dc5 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -38,7 +38,7 @@ logSys = logging.getLogger(__name__) class JailReader(ConfigReader): - actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) @@ -65,7 +65,6 @@ class JailReader(ConfigReader): ["string", "logencoding", "auto"], ["string", "backend", "auto"], ["int", "maxretry", 3], - ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], @@ -78,8 +77,10 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name, - basedir=self.getBaseDir()) + filterName, filterOpt = JailReader.extractOptions( + self.__opts["filter"]) + self.__filter = FilterReader( + filterName, self.__name, filterOpt, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -92,8 +93,9 @@ class JailReader(ConfigReader): try: if not act: # skip empty actions continue - splitAct = JailReader.splitAction(act) - action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir()) + actName, actOpt = JailReader.extractOptions(act) + action = ActionReader( + actName, self.__name, actOpt, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -105,7 +107,7 @@ class JailReader(ConfigReader): logSys.debug("Caught exception: %s" % (e,)) return False if not len(self.__actions): - logSys.warn("No actions were defined for %s" % self.__name) + logSys.warning("No actions were defined for %s" % self.__name) return True def convert(self): @@ -124,8 +126,6 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) - elif opt == "maxlines": - stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. @@ -151,23 +151,23 @@ class JailReader(ConfigReader): return stream #@staticmethod - def splitAction(action): - m = JailReader.actionCRE.match(action) + def extractOptions(option): + m = JailReader.optionCRE.match(option) d = dict() mgroups = m.groups() if len(mgroups) == 2: - action_name, action_opts = mgroups + option_name, option_opts = mgroups elif len(mgroups) == 1: - action_name, action_opts = mgroups[0], None + option_name, option_opts = mgroups[0], None else: - raise ValueError("While reading action %s we should have got up to " - "2 groups. Got: %r" % (action, mgroups)) - if not action_opts is None: + raise ValueError("While reading option %s we should have got up to " + "2 groups. Got: %r" % (option, mgroups)) + if not option_opts is None: # Huge bad hack :( This method really sucks. TODO Reimplement it. - actions = "" + options = "" escapeChar = None allowComma = False - for c in action_opts: + for c in option_opts: if c in ('"', "'") and not allowComma: # Start escapeChar = c @@ -178,20 +178,20 @@ class JailReader(ConfigReader): allowComma = False else: if c == ',' and allowComma: - actions += "" + options += "" else: - actions += c + options += c # Split using , - actionsSplit = actions.split(',') + optionsSplit = options.split(',') # Replace the tag with , - actionsSplit = [n.replace("", ',') for n in actionsSplit] + optionsSplit = [n.replace("", ',') for n in optionsSplit] - for param in actionsSplit: + for param in optionsSplit: p = param.split('=') try: d[p[0].strip()] = p[1].strip() except IndexError: - logSys.error("Invalid argument %s in '%s'" % (p, action_opts)) - return [action_name, d] - splitAction = staticmethod(splitAction) + logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) + return [option_name, d] + extractOptions = staticmethod(extractOptions) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c0373239..b7e58975 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -177,7 +177,7 @@ class Actions(JailThread): aInfo["time"] = bTicket.getTime() aInfo["matches"] = "".join(bTicket.getMatches()) if self.__banManager.addBanTicket(bTicket): - logSys.warn("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionBan(aInfo) return True @@ -217,7 +217,7 @@ class Actions(JailThread): aInfo["failures"] = ticket.getAttempt() aInfo["time"] = ticket.getTime() aInfo["matches"] = "".join(ticket.getMatches()) - logSys.warn("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionUnban(aInfo) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 8c905010..e6af4b26 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -135,7 +135,7 @@ class AsyncServer(asyncore.dispatcher): if os.path.exists(sock): logSys.error("Fail2ban seems to be already running") if force: - logSys.warn("Forcing execution of the server") + logSys.warning("Forcing execution of the server") os.remove(sock) else: raise AsyncServerException("Server already running") diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a7062f16..d4108dc2 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -632,7 +632,7 @@ class FileContainer: try: line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warn("Error decoding line from '%s' with '%s': %s" % + logSys.warning("Error decoding line from '%s' with '%s': %s" % (self.getFileName(), self.getEncoding(), `line`)) if sys.version_info >= (3,): # In python3, must be decoded line = line.decode(self.getEncoding(), 'ignore') @@ -668,11 +668,11 @@ class DNSUtils: try: return socket.gethostbyname_ex(dns)[2] except socket.gaierror: - logSys.warn("Unable to find a corresponding IP address for %s" + logSys.warning("Unable to find a corresponding IP address for %s" % dns) return list() except socket.error, e: - logSys.warn("Socket error raised trying to resolve hostname %s: %s" + logSys.warning("Socket error raised trying to resolve hostname %s: %s" % (dns, e)) return list() dnsToIp = staticmethod(dnsToIp) diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 3217e958..191b95b3 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -131,10 +131,10 @@ class FilterPoll(FileFilter): % (filename, e)) self.__file404Cnt[filename] += 1 if self.__file404Cnt[filename] > 2: - logSys.warn("Too many errors. Setting the jail idle") + logSys.warning("Too many errors. Setting the jail idle") if self.jail: self.jail.setIdle(True) else: - logSys.warn("No jail is assigned to %s" % self) + logSys.warning("No jail is assigned to %s" % self) self.__file404Cnt[filename] = 0 return False diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2d27ff6e..2a4514da 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -55,7 +55,7 @@ class Transmitter: ret = self.__commandHandler(command) ack = 0, ret except Exception, e: - logSys.warn("Command %r has failed. Received %r" + logSys.warning("Command %r has failed. Received %r" % (command, e)) ack = 1, e return ack diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d721ef00..5e79dc04 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -53,10 +53,12 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - open("%s/%s" % (self.d, fname), "w").write(""" + f = open("%s/%s" % (self.d, fname), "w") + f.write(""" [section] option = %s """ % value) + f.close() def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) @@ -112,12 +114,13 @@ class JailReaderTest(unittest.TestCase): self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'ssh-iptables') - def testSplitAction(self): + def testSplitOption(self): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitAction(action) + result = JailReader.extractOptions(action) self.assertEquals(expected, result) - + + class FilterReaderTest(unittest.TestCase): def testConvert(self): @@ -139,8 +142,9 @@ class FilterReaderTest(unittest.TestCase): "error: PAM: )?User not known to the\\nunderlying authentication." "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', - "^.+ john from host 192.168.1.1\\s*$"]] - filterReader = FilterReader("testcase01", "testcase01") + "^.+ john from host 192.168.1.1\\s*$"], + ['set', 'testcase01', 'maxlines', "1"]] + filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) @@ -148,6 +152,15 @@ class FilterReaderTest(unittest.TestCase): # Add sort as configreader uses dictionary and therefore order # is unreliable + self.assertEqual(sorted(filterReader.convert()), sorted(output)) + + filterReader = FilterReader( + "testcase01", "testcase01", {'maxlines': "5"}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + #filterReader.getOptions(["failregex", "ignoreregex"]) + filterReader.getOptions(None) + output[-1][-1] = "5" self.assertEquals(sorted(filterReader.convert()), sorted(output)) class JailsReaderTest(unittest.TestCase): diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 23f7a174..534abdbb 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -31,16 +31,19 @@ import unittest from fail2ban.server.datedetector import DateDetector from fail2ban.server.datetemplate import DateTemplate +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.__datedetector = DateDetector() self.__datedetector.addDefaultTemplate() def tearDown(self): """Call after every test case.""" + tearDownMyTime() def testGetEpochTime(self): log = "1138049999 [sshd] error: PAM: Authentication failure" diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf index 4a3a95e9..c549572d 100644 --- a/fail2ban/tests/files/filter.d/testcase01.conf +++ b/fail2ban/tests/files/filter.d/testcase01.conf @@ -32,3 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro # Values: TEXT # ignoreregex = ^.+ john from host 192.168.1.1\s*$ + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 1 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fa9340ca..e540e41e 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -34,6 +34,7 @@ from fail2ban.server.filterpoll import FilterPoll from fail2ban.server.filter import FileFilter, DNSUtils from fail2ban.server.failmanager import FailManager from fail2ban.server.failmanager import FailManagerEmpty +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -122,7 +123,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, found, output, count) -def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): """Copy lines from one file to another (which might be already open) Returns open fout @@ -131,8 +132,10 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # on old Python st_mtime is int, so we should give at least 1 sec so # polling filter could detect the change time.sleep(1) - if isinstance(fin, str): # pragma: no branch - only used with str in test cases - fin = open(fin, 'r') + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + fin = open(in_, 'r') + else: + fin = in_ # Skip for i in xrange(skip): _ = fin.readline() @@ -150,6 +153,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line fout = open(fout, mode) fout.write('\n'.join(lines)) fout.flush() + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + # Opened earlier, therefore must close it + fin.close() # to give other threads possibly some time to crunch time.sleep(0.1) return fout @@ -213,6 +219,7 @@ class LogFileMonitor(unittest.TestCase): """ def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') @@ -222,6 +229,7 @@ class LogFileMonitor(unittest.TestCase): self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): + tearDownMyTime() _killfile(self.file, self.name) pass @@ -288,7 +296,7 @@ class LogFileMonitor(unittest.TestCase): # # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -305,6 +313,7 @@ class LogFileMonitor(unittest.TestCase): def testNewChangeViaGetFailures_move(self): # # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') self.filter.getFailures(self.name) @@ -313,7 +322,7 @@ class LogFileMonitor(unittest.TestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -363,6 +372,7 @@ def get_monitor_failures_testcase(Filter_): count = 0 def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' self.name = '%s-%d' % (testclass_name, self.count) MonitorFailures.count += 1 # so we have unique filenames across tests @@ -380,6 +390,7 @@ def get_monitor_failures_testcase(Filter_): def tearDown(self): + tearDownMyTime() #print "D: SLEEPING A BIT" #import time; time.sleep(5) #print "D: TEARING DOWN" @@ -449,7 +460,7 @@ def get_monitor_failures_testcase(Filter_): def test_rewrite_file(self): # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # What if file gets overridden @@ -463,6 +474,7 @@ def get_monitor_failures_testcase(Filter_): def test_move_file(self): # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time @@ -472,25 +484,25 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) def test_new_bogus_file(self): # to make sure that watching whole directory does not effect - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # create a bogus file in the same directory and see if that doesn't affect - open(self.name + '.bak2', 'w').write('') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + open(self.name + '.bak2', 'w').close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') @@ -543,6 +555,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = FileFilter(None) self.filter.setActive(True) # TODO Test this @@ -551,6 +564,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" + tearDownMyTime() diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 6b894193..95646d63 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,9 +22,13 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback +import logging, os, re, traceback, time, unittest from os.path import basename, dirname +from fail2ban.server.mytime import MyTime + +logSys = logging.getLogger(__name__) + # # Following "traceback" functions are adopted from PyMVPA distributed # under MIT/Expat and copyright by PyMVPA developers (i.e. me and @@ -99,3 +103,105 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + +old_TZ = os.environ.get('TZ', None) +def setUpMyTime(): + # Set the time to a fixed, known value + # Sun Aug 14 12:00:00 CEST 2005 + # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match + os.environ['TZ'] = 'Europe/Zurich' + time.tzset() + MyTime.setTime(1124013600) + +def tearDownMyTime(): + os.environ.pop('TZ') + if old_TZ: + os.environ['TZ'] = old_TZ + time.tzset() + MyTime.myTime = None + +from fail2ban.tests import banmanagertestcase +from fail2ban.tests import clientreadertestcase +from fail2ban.tests import failmanagertestcase +from fail2ban.tests import filtertestcase +from fail2ban.tests import servertestcase +from fail2ban.tests import datedetectortestcase +from fail2ban.tests import actiontestcase +from fail2ban.tests import sockettestcase + +def gatherTests(regexps=None, no_network=False): + if not regexps: # pragma: no cover + tests = unittest.TestSuite() + else: # pragma: no cover + import re + class FilteredTestSuite(unittest.TestSuite): + _regexps = [re.compile(r) for r in regexps] + def addTest(self, suite): + suite_str = str(suite) + for r in self._regexps: + if r.search(suite_str): + super(FilteredTestSuite, self).addTest(suite) + return + + tests = FilteredTestSuite() + + # Server + #tests.addTest(unittest.makeSuite(servertestcase.StartStop)) + tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) + tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) + # FailManager + tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) + # BanManager + tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + # ClientReaders + tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + # CSocket and AsyncServer + tests.addTest(unittest.makeSuite(sockettestcase.Socket)) + + # Filter + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) + tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) + + # DateDetector + tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) + + # + # Extensive use-tests of different available filters backends + # + + from fail2ban.server.filterpoll import FilterPoll + filters = [FilterPoll] # always available + + # Additional filters available only if external modules are available + # yoh: Since I do not know better way for parametric tests + # with good old unittest + try: + from fail2ban.server.filtergamin import FilterGamin + filters.append(FilterGamin) + except Exception, e: # pragma: no cover + logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e) + + try: + from fail2ban.server.filterpyinotify import FilterPyinotify + filters.append(FilterPyinotify) + except Exception, e: # pragma: no cover + logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e) + + for Filter_ in filters: + tests.addTest(unittest.makeSuite( + filtertestcase.get_monitor_failures_testcase(Filter_))) + + # Server test for logging elements which break logging used to support + # testcases analysis + tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) + + return tests diff --git a/files/fail2ban-tmpfiles.conf b/files/fail2ban-tmpfiles.conf new file mode 100644 index 00000000..3fd783f3 --- /dev/null +++ b/files/fail2ban-tmpfiles.conf @@ -0,0 +1 @@ +D /var/run/fail2ban 0755 root root - \ No newline at end of file diff --git a/files/fail2ban.service b/files/fail2ban.service new file mode 100644 index 00000000..9c44042b --- /dev/null +++ b/files/fail2ban.service @@ -0,0 +1,14 @@ +[Unit] +Description=Fail2ban Service +After=syslog.target network.target + +[Service] +Type=forking +ExecStart=/usr/bin/fail2ban-client -x start +ExecStop=/usr/bin/fail2ban-client stop +ExecReload=/usr/bin/fail2ban-client reload +PIDFile=/var/run/fail2ban/fail2ban.pid +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 552b0ac0..d571dc7b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -140,6 +140,11 @@ Using Python "string interpolation" mechanisms, other definitions are allowed an baduseragents = IE|wget failregex = useragent=%(baduseragents)s +.PP +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +.TP +\fBmaxlines\fR +specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .PP Filters can also have a section called [INCLUDES]. This is used to read other configuration files. diff --git a/setup.py b/setup.py index 57b75225..058c4b21 100755 --- a/setup.py +++ b/setup.py @@ -18,11 +18,17 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Steven Hiscocks, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2013 Fail2Ban Contributors" __license__ = "GPL" -from distutils.core import setup +try: + import setuptools + from setuptools import setup +except ImportError: + setuptools = None + from distutils.core import setup + try: # python 3.x from distutils.command.build_py import build_py_2to3 as build_py @@ -33,10 +39,27 @@ except ImportError: from distutils.command.build_py import build_py from distutils.command.build_scripts import build_scripts from os.path import isfile, join, isdir -import sys +import sys, warnings from glob import glob -from fail2ban.version import version +if setuptools and "test" in sys.argv: + import logging + logSys = logging.getLogger("fail2ban") + hdlr = logging.StreamHandler(sys.stdout) + fmt = logging.Formatter("%(asctime)-15s %(message)s") + hdlr.setFormatter(fmt) + logSys.addHandler(hdlr) + if set(["-q", "--quiet"]) & set(sys.argv): + logSys.setLevel(logging.FATAL) + warnings.simplefilter("ignore") + sys.warnoptions.append("ignore") + elif set(["-v", "--verbose"]) & set(sys.argv): + logSys.setLevel(logging.DEBUG) + else: + logSys.setLevel(logging.INFO) +elif "test" in sys.argv: + print("python distribute required to execute fail2ban tests") + print("") longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or @@ -45,12 +68,26 @@ too many password failures. It updates firewall rules to reject the IP address or executes user defined commands.''' +if setuptools: + setup_extra = { + 'test_suite': "fail2ban.tests.utils.gatherTests", + 'use_2to3': True, + } +else: + setup_extra = {} + +# Get version number, avoiding importing fail2ban. +# This is due to tests not functioning for python3 as 2to3 takes place later +f = open(join("fail2ban", "version.py")) +exec(f.read()) +f.close() + setup( name = "fail2ban", version = version, description = "Ban IPs that make too many password failures", long_description = longdesc, - author = "Cyril Jaquier", + author = "Cyril Jaquier & Fail2Ban Contributors", author_email = "cyril.jaquier@fail2ban.org", url = "http://www.fail2ban.org", license = "GPL", @@ -88,7 +125,8 @@ setup( ('/usr/share/doc/fail2ban', ['README', 'DEVELOP', 'doc/run-rootless.txt'] ) - ] + ], + **setup_extra ) # Do some checks after installation