diff --git a/ChangeLog b/ChangeLog index 5c5ffc99..db89a39e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,10 +47,13 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests * files/redhat-initd - rewritten to use stock init.d functions thus avoiding problems with getpid. Also $network and iptables moved to Should- rc init fields + - New Features: Andy Fragen and Daniel Black * filter.d/osx-ipfw.conf - ipfw action for OSX based on random rule numbers. + Anonymous: + * action.d/osx-afctl - an action based on afctl for osx Daniel Black & ykimon * filter.d/3proxy.conf -- filter added Daniel Black @@ -59,6 +62,10 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests * add date expression for apache-2.4 - milliseconds Christophe Carles & Daniel Black * filter.d/perdition.conf -- filter added + Mark McKinstry + * action.d/apf.conf - add action for Advanced Policy Firewall (apf) + Amir Caspi and kjohnsonecl + * filter.d/uwimap-auth - filter for uwimap-auth IMAP/POP server - Enhancements: François Boulogne and Frédéric @@ -74,6 +81,8 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests * filter.d/gssftpd - anchored regex at start * filter.d/mysqld-auth.conf - mysql can use syslog * fail2ban-regex - now generated http://www.debuggex.com urls for debugging + * filter.d/sshd - regex enhancements to support openssh-6.3. Closes Debian + bug #722970 Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs diff --git a/DEVELOP b/DEVELOP index 61248393..8f6d7a19 100644 --- a/DEVELOP +++ b/DEVELOP @@ -1,6 +1,6 @@ - __ _ _ ___ _ - / _|__ _(_) |_ ) |__ __ _ _ _ - | _/ _` | | |/ /| '_ \/ _` | ' \ + __ _ _ ___ _ + / _|__ _(_) |_ ) |__ __ _ _ _ + | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ @@ -26,7 +26,7 @@ Pull Requests When submitting pull requests on GitHub we ask you to: * Clearly describe the problem you're solving; -* Don't introduce regressions that will make it hard for systems adminstrators +* Don't introduce regressions that will make it hard for systems administrators to update; * If adding a major feature rebase your changes on master and get to a single commit; * Include test cases (see below); @@ -37,49 +37,304 @@ When submitting pull requests on GitHub we ask you to: Filters ======= -* Include sample logs with 1.2.3.4 used for IP addresses and - example.com/example.org used for DNS names -* Ensure sample log is provided in testcases/files/logs/ with same name as the - filter. Each log line should include match meta data for time & IP above - every line (see other sample log files for examples) -* Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if - \d+ is sufficient -* Use the functionality of regexs http://docs.python.org/2/library/re.html -* Take a look at the source code of the application. You may see optional or - extra log messages, or parts there of, that need to form part of your regex. +Filters are tricky. They need to: +* work with a variety of the versions of the software that generates the logs; +* work with the range of logging configuration options available in the + software; +* work with multiple operating systems; +* not make assumptions about the log format in excess of the software + (e.g. do not assume a username doesn't contain spaces and use \S+ unless + you've checked the source code); +* account for how future versions of the software will log messages + (e.g. guess what would happen to the log message if different authentication + types are added); +* not be susceptible to DoS vulnerabilities (see Filter Security below); and +* match intended log lines only. -If you only have a basic knowledge of regular repressions read -http://docs.python.org/2/library/re.html first. +Please follow the steps from Filter Test Cases to Developing Filter Regular +Expressions and submit a GitHub pull request (PR) afterwards. If you get stuck, +you can push your unfinished changes and still submit a PR -- describe +what you have done, what is the hurdle, and we'll attempt to help (PR +will be automagically updated with future commits you would push to +complete it). + +Filter test cases +----------------- + +Purpose: + +Start by finding the log messages that the application generates related to +some form of authentication failure. If you are adding to an existing filter +think about whether the log messages are of a similar importance and purpose +to the existing filter. If you were a user of Fail2Ban, and did a package +update of Fail2Ban that started matching new log messages, would anything +unexpected happen? Would the bantime/findtime for the jail be appropriate for +the new log messages? If it doesn't, perhaps it needs to be in a separate +filter definition, for example like exim filter aims at authentication failures +and exim-spam at log messages related to spam. + +Even if it is a new filter you may consider separating the log messages into +different filters based on purpose. + +Cause: + +Are some of the log lines a result of the same action? For example, is a PAM +failure log message, followed by an application specific failure message the +result of the same user/script action? If you add regular expressions for +both you would end up with two failures for a single action. +Therefore, select the most appropriate log message and document the other log +message) with a test case not to match it and a description as to why you chose +one over another. + +With the selected log lines consider what action has caused those log +messages and whether they could have been generated by accident? Could +the log message be occurring due to the first step towards the application +asking for authentication? Could the log messages occur often? If some of +these are true make a note of this in the jail.conf example that you provide. + +Samples: + +It is important to include log file samples so any future change in the regular +expression will still work with the log lines you have identified. + +The sample log messages are provided in a file under testcases/files/logs/ +named identically as the corresponding filter (but without .conf extension). +Each log line should be preceded by a line with failJSON metadata (so the logs +lines are tested in the test suite) directly above the log line. If there is +any specific information about the log message, such as version or an +application configuration option that is needed for the message to occur, +include this in a comment (line beginning with #) above the failJSON metadata. + +Log samples should include only one, definitely not more than 3, examples of +log messages of the same form. If log messages are different in different +versions of the application log messages that show this are encouraged. + +Also attempt to inject an IP into the application (e.g. by specifying +it as a username) so that Fail2Ban possibly detects the IP +from user input rather than the true origin. See the Filter Security section +and the top example in testcases/files/logs/apache-auth as to how to do this. +One you have discovered that this is possible, correct the regex so it doesn't +match and provide this as a test case with "match": false (see failJSON below). + +If the mechanism to create the log message isn't obvious provide a +configuration and/or sample scripts testcases/files/config/{filtername} and +reference these in the comments above the log line. + +FailJSON metadata: + +A failJSON metadata is a comment immediately above the log message. It will +look like: + +# failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "93.184.216.119" } + +Time should match the time of the log message. It is in a specific format of +Year-Month-Day'T'Hour:minute:Second. If your log message does not include a +year, like the example below, the year should be listed as 2005, if before Sun +Aug 14 10am UTC, and 2004 if afterwards. Here is an example failJSON +line preceding a sample log line: + +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 + +The "host" in failJSON should contain the IP or domain that should be blocked. + +For long lines that you do not want to be matched (e.g. from log injection +attacks) and any log lines to be excluded (see "Cause" section above), set +"match": false in the failJSON and describe the reason in the comment above. + +After developing regexes, the following command will test all failJSON metadata +against the log lines in all sample log files + +./fail2ban-testcases testSampleRegex + +Developing Filter Regular Expressions +------------------------------------- + +Date/Time: + +At the moment, Fail2Ban depends on log lines to have time stamps. That is why +before starting to develop failregex, check if your log line format known to +Fail2Ban. Copy the time component from the log line and append an IP address to +test with following command: + +./fail2ban-regex "2013-09-19 02:46:12 1.2.3.4" "" + +Output of such command should contain something like: + +Date template hits: +|- [# of hits] date format +| [1] Year-Month-Day Hour:Minute:Second + +Ensure that the template description matches time/date elements in your log line +time stamp. If there is no matched format then date template needs to be added +to server/datedetector.py. Ensure that a new template is added in the order +that more specific matches occur first and that there is no confusion between a +Day and a Month. + +Filter file: + +The filter is specified in a config/filter.d/{filtername}.conf file. Filter file +can have sections INCLUDES (optional) and Definition as follows: + +[INCLUDES] + +before = common.conf + +after = filtername.local + +[Definition] + +failregex = .... + +ignoreregex = .... + +This is also documented in the man page jail.conf (section 5). Other definitions +can be added to make failregex's more readable and maintainable to be used +through string Interpolations (see http://docs.python.org/2.7/library/configparser.html) + + +General rules: + +Use "before" if you need to include a common set of rules, like syslog or if +there is a common set of regexes for multiple filters. + +Use "after" if you wish to allow the user to overwrite a set of customisations +of the current filter. This file doesn't need to exist. + +Try to avoid using ignoreregex mainly for performance reasons. The case when you +would use it is if in trying to avoid using it, you end up with an unreadable +failregex. + +Syslog: + +If your application logs to syslog you can take advantage of log line prefix +definitions present in common.conf. So as a base use: + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = app + +failregex = ^%(__prefix_line)s + +In this example common.conf defines __prefix_line which also contains the +_daemon name (in syslog terms the service) you have just specified. _daemon +can also be a regex. + +For example, to capture following line _daemon should be set to "dovecot" + +Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 + +and then ^%(__prefix_line)s would match "Dec 12 11:19:11 dunnart dovecot: +". Note it matches the trailing space(s) as well. + +Substitutions (AKA string interpolations): + +We have used string interpolations in above examples. They are useful for +making the regexes more readable, reuse generic patterns in multiple failregex +lines, and also to refer definition of regex parts to specific filters or even +to the user. General principle is that value of a _name variable replaces +occurrences of %(_name)s within the same section or anywhere in the config file +if defined in [DEFAULT] section. + +Regular Expressions: + +Regular expressions (failregex, ignoreregex) assume that the date/time has been +removed from the log line (this is just how fail2ban works internally ATM). + +If the format is like ' error 1.2.3.4 is evil' then you need to match +the < at the start so regex should be similar to '^<> is evil$' using + where the IP/domain name appears in the log line. + +The following general rules apply to regular expressions: + +* ensure regexes start with a ^ and are as restrictive as possible. E.g. do not + use .* if \d+ is sufficient; +* use functionality of Python regexes defined in the standard Python re library + http://docs.python.org/2/library/re.html; +* make regular expressions readable (as much as possible). E.g. + (?:...) represents a non-capturing regex but (...) is more readable, thus + preferred. + +If you have only a basic knowledge of regular repressions we advise to read +http://docs.python.org/2/library/re.html first. It doesn't take long and would +remind you e.g. which characters you need to escape and which you don't. + +Developing/testing a regex: + +You can develop a regex in a file or using command line depending on your +preference. You can also use samples you have already created in the test cases +or test them one at a time. + +The general tool for testing Fail2Ban regexes is fail2ban-regex. To see how to +use it run: + +./fail2ban-regex --help + +Take note of -l heavydebug / -l debug and -v as they might be very useful. + +TIP: Take a look at the source code of the application you are developing + failregex for. You may see optional or extra log messages, or parts there + of, that need to form part of your regex. It may also reveal how some + parts are constrained and different formats depending on configuration or + less common usages. + +TIP: Some applications log spaces at the end. If you are not sure add \s*$ as + the end part of the regex. + +If your regex is not matching, http://www.debuggex.com/?flavor=python can help +to tune it: + +* use regex from the ./fail2ban-regex output (to ensure all substitutions are +done) and replace with (?&.ipv4). Make sure that regex type set to +Python; +* for the test data put your log output with the time removed; +- when you have fixed the regex put it back into your filter file. + +Please spread the good word about debuggex - Serge Toarca is kindly continuing +its free availability to Open Source developers. + +Finishing up: + +If you've added a new filter, add a new entry in config/jail.conf. The theory +here is that a user will create a jail.local with [filtername]\nenable=true to +enable your jail. + +So more specifically in the [filter] section in jail.conf: +* ensure that you have "enabled = false" (users will enable as needed); +* use "filter =" set to your filter name; +* use a typical action to disable ports associated with the application; +* set "logpath" to the usual location of application log file; +* if the default findtime or bantime isn't appropriate to the filter, specify + more appropriate choices (possibly with a brief comment line). + +Submit github pull request (See "Pull Requests" above) for +github.com/fail2ban/fail2ban containing your great work. Filter Security --------------- -Poor filter regular expressions are suseptable to DoS attacks. +Poor filter regular expressions are susceptible to DoS attacks. -When a remote user has the ability to introduce text that will match the -filter regex, such that the inserted text matches the part, they have the +When a remote user has the ability to introduce text that would match filter's +failregex, while matching inserted text to the part, they have the ability to deny any host they choose. -So the part must be anchored on text generated by the application, and not -the user, to a sufficient extent that the user cannot insert the entire text. +So the part must be anchored on text generated by the application, and +not the user, to a extent sufficient to prevent user inserting the entire text +matching this or any other failregex. -Filters are matched against the log line with their date removed. - -Ideally filter regex should anchor to the beginning and end of the log line -however as more applications log at the beginning than the end, achoring the +Ideally filter regex should anchor at the beginning and at the end of log line. +However as more applications log at the beginning than the end, anchoring the beginning is more important. If the log file used by the application is shared -with other applications, like system logs, ensure the other application that -use that log file do not log user generated text at the beginning of the line, -or, if they do, ensure the regexs of the filter are sufficient to mitigate the -risk of insertion. +with other applications, like system logs, ensure the other application that use +that log file do not log user generated text at the beginning of the line, or, +if they do, ensure the regexes of the filter are sufficient to mitigate the risk +of insertion. -When creating a regex that extends back to the begining remember the date part -has been removed within fail2ban so theres no need to match that. If the format -is like ' error 1.2.3.4 is evil' then you will need to match the < at -the start so here the regex would start like '^<> is evil$'. - -Some applications log spaces at the end. If you're not sure add \s*$ as the -end part of the regex. Examples of poor filters ------------------------ @@ -96,13 +351,13 @@ We make a failregex Now think evil. The user does the command 'blah from 1.2.3.44' -The program diliently logs: +The program diligently logs: Apr-07-13 07:08:36 Invalid command blah from 1.2.3.44 from 1.2.3.4 And fail2ban matches 1.2.3.44 as the IP that it ban. A DoS attack was successful. -The fix here is that the command can be anything so .* is approprate. +The fix here is that the command can be anything so .* is appropriate. ^Invalid command .* from @@ -121,10 +376,10 @@ banned. 2. Filter regex can match other user injected data -From the apache vulnerability CVE-2013-2178 +From the Apache vulnerability CVE-2013-2178 ( original ref: https://vndh.net/note:fail2ban-089-denial-service ). -An example bad regex for apache: +An example bad regex for Apache: failregex = [[]client []] user .* not found @@ -140,10 +395,10 @@ Now the log line will be: As this log line doesn't match other expressions hence it matches the above regex and blocks 192.168.33.1 as a denial of service from the HTTP requester. -3. Applicaiton generates two identical log messages with different meanings +3. Application generates two identical log messages with different meanings If the application generates the following two messages under different -circmstances: +circumstances: client : authentication failed client : authentication failed @@ -179,7 +434,7 @@ coverage run fail2ban-testcases coverage html Then look at htmlcov/index.html and see how much coverage your test cases -exert over the codebase. Full coverage is a good thing however it may not be +exert over the code base. Full coverage is a good thing however it may not be complete. Try to ensure tests cover as many independent paths through the code. @@ -270,7 +525,7 @@ Design Fail2Ban was initially developed with Python 2.3 (IIRC). It should still be compatible with Python 2.4 and such compatibility assurance makes code ... old-fashioned in many places (RF-Note). In 0.7 the -design went through major refactoring into client/server, +design went through major re-factoring into client/server, a-thread-per-jail design which made it a bit difficult to follow. Below you can find a sketchy description of the main components of the system to orient yourself better. @@ -381,7 +636,7 @@ one way or another provide except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) -thus channeling "ban tickets" from their failManager to the +thus channelling "ban tickets" from their failManager to the corresponding jail. action.py @@ -473,11 +728,11 @@ ver. 0.8.12 (2013/XX/XXX) - wanna-be-released ----------- - Fixes: - + - New Features: - + - Enhancements: - + and adjust common/version.py to carry .dev suffix to signal a version under development. diff --git a/THANKS b/THANKS index ad8eed60..e70ca9c9 100644 --- a/THANKS +++ b/THANKS @@ -8,6 +8,7 @@ be added Adrien Clerc ache +Amir Caspi Andrey G. Grozin Andy Fragen Arturo 'Buanzo' Busleiman @@ -38,10 +39,12 @@ Joël Bertrand JP Espinosa Justin Shore Kévin Drapel +kjohnsonecl kojiro Manuel Arostegui Ramirez Marcel Dopita Mark Edgington +Mark McKinstry Markus Hoffmann Marvin Rouge mEDI diff --git a/client/jailreader.py b/client/jailreader.py index 1a2be432..7fbac423 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, re, glob +import logging, re, glob, os.path from configreader import ConfigReader from filterreader import FilterReader @@ -55,7 +55,23 @@ class JailReader(ConfigReader): def isEnabled(self): return self.__force_enable or self.__opts["enabled"] - + + @staticmethod + def _glob(path): + """Given a path for glob return list of files to be passed to server. + + Dangling symlinks are warned about and not returned + """ + pathList = [] + for p in glob.glob(path): + if not os.path.exists(p): + logSys.warning("File %s doesn't even exist, thus cannot be monitored" % p) + elif not os.path.lexists(p): + logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) + else: + pathList.append(p) + return pathList + def getOptions(self): opts = [["bool", "enabled", "false"], ["string", "logpath", "/var/log/messages"], @@ -118,7 +134,7 @@ class JailReader(ConfigReader): if opt == "logpath": found_files = 0 for path in self.__opts[opt].split("\n"): - pathList = glob.glob(path) + pathList = JailReader._glob(path) if len(pathList) == 0: logSys.error("No file(s) found for glob %s" % path) for p in pathList: diff --git a/config/action.d/apf.conf b/config/action.d/apf.conf new file mode 100644 index 00000000..9af3066d --- /dev/null +++ b/config/action.d/apf.conf @@ -0,0 +1,43 @@ +# Fail2Ban configuration file +# +# Author: Mark McKinstry +# +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# number of failures +#