From 6884593ab825068cdecc55658a9213513ca6ed16 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:15:20 +0100 Subject: [PATCH 01/15] New filter `nginx-limit-req` ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) --- ChangeLog | 7 ++-- config/filter.d/nginx-limit-req.conf | 39 +++++++++++++++++++++++ config/jail.conf | 8 +++++ fail2ban/tests/files/logs/nginx-limit-req | 6 ++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 config/filter.d/nginx-limit-req.conf create mode 100644 fail2ban/tests/files/logs/nginx-limit-req diff --git a/ChangeLog b/ChangeLog index 391eabc6..cd1ce7bd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,11 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) - New Features: + * New filters: + - openhab - domotic software authentication failure with the + rest api and web interface (gh-1223) + - nginx-limit-req - ban hosts, that were failed through nginx by limit + request processing rate (ngx_http_limit_req_module) - Enhancements: * Do not rotate empty log files @@ -37,8 +42,6 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) - * Added filter for openhab domotic software authentication failure with the - rest api and web interface (gh-1223) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf new file mode 100644 index 00000000..eb804798 --- /dev/null +++ b/config/filter.d/nginx-limit-req.conf @@ -0,0 +1,39 @@ +# Fail2ban filter configuration for nginx :: limit_req +# used to ban hosts, that were failed through nginx by limit request processing rate +# +# Author: Serg G. Brester (sebres) +# +# To use 'nginx-limit-req' filter you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# +# Example: +# +# http { +# ... +# limit_req_zone $binary_remote_addr zone=lr_zone:10m rate=1r/s; +# ... +# # http, server, or location: +# location ... { +# limit_req zone=lr_zone burst=1 nodelay; +# ... +# } +# ... +# } +# ... +# + +[Definition] + +# Specify following expression to define exact zones, if you want to ban IPs limited +# from specified zones only. +# Example: +# +# ngx_limit_req_zones = lr_zone|lr_zone2 +# +ngx_limit_req_zones = [^"]+ + +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +ignoreregex = + diff --git a/config/jail.conf b/config/jail.conf index fd7f376e..fc175550 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -317,6 +317,14 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# or for example see in 'config/filter.d/nginx-limit-req.conf' +[nginx-limit-req] +port = http,https +logpath = %(nginx_error_log)s + [nginx-botsearch] port = http,https diff --git a/fail2ban/tests/files/logs/nginx-limit-req b/fail2ban/tests/files/logs/nginx-limit-req new file mode 100644 index 00000000..68f1b239 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-limit-req @@ -0,0 +1,6 @@ + +# failJSON: { "time": "2015-10-29T20:01:02", "match": true , "host": "1.2.3.4" } +2015/10/29 20:01:02 [error] 256554#0: *99927 limiting requests, excess: 1.852 by zone "one", client: 1.2.3.4, server: example.com, request: "POST /index.htm HTTP/1.0", host: "exmaple.com" + +# failJSON: { "time": "2015-10-29T19:24:05", "match": true , "host": "192.0.2.0" } +2015/10/29 19:24:05 [error] 12684#12684: *22174 limiting requests, excess: 1.495 by zone "one", client: 192.0.2.0, server: example.com, request: "GET /index.php HTTP/1.1", host: "example.com", referrer: "https://example.com" From 53b39162a1b2d026bf0e9eb14f3ffa61f5cc1f8a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:55:23 +0100 Subject: [PATCH 02/15] Shortly, much faster and stable version of regexp (possible because expression is start-anchored and does not contains closely to catch-all sub expressions) --- config/filter.d/nginx-limit-req.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index eb804798..589d3d78 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -33,7 +33,13 @@ # ngx_limit_req_zones = [^"]+ -failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ +# Use following full expression if you should range limit request to specified +# servers, requests, referrers etc. only : +# +# failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +# Shortly, much faster and stable version of regexp: +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: ignoreregex = From f359ed8c367a97060afcc54164263bfad9ed92d9 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 15:36:18 +0100 Subject: [PATCH 03/15] Fixed directly defined banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (+ man entries for both variables added); closes gh-1216 --- ChangeLog | 2 ++ config/jail.conf | 7 ++++--- man/jail.conf.5 | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..beaba762 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) * Fix jail.conf.5 man's section (gh-1226) + * Fixed default banaction for allports jails like pam-generic, recidive, etc + with new default variable `banaction_allports` (gh-1216) - New Features: * New filters: diff --git a/config/jail.conf b/config/jail.conf index fc175550..69fef818 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -154,6 +154,7 @@ port = 0:65535 # action_* variables. Can be overridden globally or per # section within jail.local file banaction = iptables-multiport +banaction_allports = iptables-allports # The simplest action to take: ban only action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] @@ -713,7 +714,7 @@ maxretry = 5 [recidive] logpath = /var/log/fail2ban.log -banaction = iptables-allports +banaction = %(banaction_allports)s bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 @@ -724,7 +725,7 @@ maxretry = 5 [pam-generic] # pam-generic filter can be customized to monitor specific subset of 'tty's -banaction = iptables-allports +banaction = %(banaction_allports)s logpath = %(syslog_authpriv)s @@ -770,7 +771,7 @@ maxretry = 1 enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current maxretry = 6 -banaction = iptables-allports +banaction = %(banaction_allports)s [directadmin] enabled = false diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 957a04b4..5dc64a4b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -146,6 +146,12 @@ Ensure syslog or the program that generates the log file isn't configured to com .B logencoding encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP +.B banaction +default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +.TP +.B banaction_allports +default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +.TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS From e825e977ccb897ddeadd573854e2666ee8963038 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 17:51:30 +0100 Subject: [PATCH 04/15] Nginx log paths extended (prefixed with "*" wildcard) closes gh-1237 --- ChangeLog | 1 + config/paths-common.conf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..66805c36 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) + * Nginx log paths extended (prefixed with "*" wildcard) (gh-1237) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/paths-common.conf b/config/paths-common.conf index bf3cfb6a..1aac027b 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -24,9 +24,9 @@ auditd_log = /var/log/audit/audit.log exim_main_log = /var/log/exim/mainlog -nginx_error_log = /var/log/nginx/error.log +nginx_error_log = /var/log/nginx/*error.log -nginx_access_log = /var/log/nginx/access.log +nginx_access_log = /var/log/nginx/*access.log lighttpd_error_log = /var/log/lighttpd/error.log From fcf03790f4377f4083ca114fffc8aea5e67685ee Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 1 Nov 2015 16:55:45 +0100 Subject: [PATCH 05/15] fixed misleading documentation of `banaction` --- man/jail.conf.5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5dc64a4b..51ea7097 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -147,10 +147,10 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports -default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: From b40c6cbd9a0c1b51e0cc0f0c4629b84d60c2c129 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 1 Nov 2015 11:28:58 -0500 Subject: [PATCH 06/15] ENH: .mailmap file to bring some names together for git shortlog -sn --- .mailmap | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..9b7cce3a --- /dev/null +++ b/.mailmap @@ -0,0 +1,5 @@ +Lee Clemens +Serg G. Brester +Serg G. Brester +Serg G. Brester +Viktor Szépe From 94cffece12dc2ff0efd6a8b6761b079eb9491c68 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 2 Nov 2015 21:19:15 +0100 Subject: [PATCH 07/15] New interpolation feature for definition config readers - ``, as extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters; --- ChangeLog | 7 ++++ fail2ban/client/configreader.py | 4 +- fail2ban/tests/clientreadertestcase.py | 57 ++++++++++++++++++-------- man/jail.conf.5 | 30 ++++++++++++-- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..151b5a9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,13 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released with new default variable `banaction_allports` (gh-1216) - New Features: + * New interpolation feature for definition config readers - `` + (means last known init definition of filters or actions with name `parameter`). + This interpolation makes possible to extend a parameters of stock filter or + action directly in jail inside jail.local file, without creating a separately + filter.d/*.local file. + As extension to interpolation `%(known/parameter)s`, that does not works for + filter and action init parameters * New filters: - openhab - domotic software authentication failure with the rest api and web interface (gh-1223) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d5675cc8..c6dd1b60 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -285,8 +285,10 @@ class DefinitionInitConfigReader(ConfigReader): if self.has_section("Init"): for opt in self.options("Init"): + v = self.get("Init", opt) + self._initOpts['known/'+opt] = v if not opt in self._initOpts: - self._initOpts[opt] = self.get("Init", opt) + self._initOpts[opt] = v def convert(self): raise NotImplementedError diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 94fe1828..d19090be 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -165,11 +165,11 @@ class JailReaderTest(LogCaptureTestCase): self.__share_cfg = {} def testIncorrectJail(self): - jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg) + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg) self.assertRaises(ValueError, jail.read) def testJailActionEmpty(self): - jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -177,7 +177,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): - jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -200,7 +200,7 @@ class JailReaderTest(LogCaptureTestCase): if STOCK: def testStockSSHJail(self): - jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -274,6 +274,10 @@ class JailReaderTest(LogCaptureTestCase): class FilterReaderTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(FilterReaderTest, self).__init__(*args, **kwargs) + self.__share_cfg = {} + def testConvert(self): output = [['set', 'testcase01', 'addfailregex', "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" @@ -311,9 +315,8 @@ class FilterReaderTest(unittest.TestCase): # is unreliable self.assertEqual(sorted(filterReader.convert()), sorted(output)) - filterReader = FilterReader( - "testcase01", "testcase01", {'maxlines': "5"}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) @@ -322,8 +325,8 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -331,16 +334,34 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertEqual(sorted(c), sorted(output)) + + def testFilterReaderSubstitionKnown(self): + output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=']] + filterName, filterOpt = JailReader.extractOptions( + 'substition[honeypot=",", sweet="test,,test2"]') + filterReader = FilterReader('substition', "jailname", filterOpt, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() self.assertEqual(sorted(c), sorted(output)) def testFilterReaderSubstitionFail(self): - filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}) - filterReader.setBaseDir(TEST_FILES_DIR) + # directly subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + self.assertRaises(ValueError, FilterReader.convert, filterReader) + # cross subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) @@ -508,12 +529,13 @@ class JailsReaderTest(LogCaptureTestCase): if jail == 'INCLUDES': continue filterName = jails.get(jail, 'filter') + filterName, filterOpt = JailReader.extractOptions(filterName) allFilters.add(filterName) self.assertTrue(len(filterName)) # moreover we must have a file for it # and it must be readable as a Filter - filterReader = FilterReader(filterName, jail, {}) - filterReader.setBaseDir(CONFIG_DIR) + filterReader = FilterReader(filterName, jail, filterOpt, + share_config=self.__share_cfg, basedir=CONFIG_DIR) self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName) # opens fine filterReader.getOptions({}) # reads fine @@ -551,7 +573,10 @@ class JailsReaderTest(LogCaptureTestCase): filters = set(os.path.splitext(os.path.split(a)[1])[0] for a in glob.glob(os.path.join('config', 'filter.d', '*.conf')) if not a.endswith('common.conf')) - filters_jail = set(jail.options['filter'] for jail in jails.jails) + # get filters of all jails (filter names without options inside filter[...]) + filters_jail = set( + JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails + ) self.maxDiff = None self.assertTrue(filters.issubset(filters_jail), "More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail)) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 51ea7097..7a31e8b1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -89,13 +89,33 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. .RS +.nf baduseragents = IE|wget +failregex = %(known/failregex)s + useragent=%(baduseragents)s +.fi .RE + +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. + .RS -failregex = useragent=%(baduseragents)s +# filter.d/test.conf: +.nf +[Init] +test.method = GET +baduseragents = IE|wget +[Definition] +failregex = ^%(__prefix_line)\\s+""\\s+test\\s+regexp\\s+-\\s+useragent=(?:) + +# jail.local: +[test] +# use filter "test", overwrite method to "POST" and extend known bad agents with "badagent": +filter = test[test.method=POST, baduseragents="badagent|"] +.fi .RE Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. @@ -253,7 +273,7 @@ The maximum period of time in seconds that a command can executed, before being Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed. @@ -312,7 +332,7 @@ is the regex to identify log entries that should be ignored by Fail2Ban, even if .PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. @@ -327,6 +347,8 @@ Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO860 \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP +Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. + Filters can also have a section called [INCLUDES]. This is used to read other configuration files. .TP From ba76f4ca2fa058ca187a2990d3ff0b66b6211940 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 2 Nov 2015 15:21:14 -0700 Subject: [PATCH 08/15] Fix typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 69fef818..3bb17e0a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -80,7 +80,7 @@ maxretry = 5 # auto: will try to use the following backends, in order: # pyinotify, gamin, polling. # -# Note: if systemd backend is choses as the default but you enable a jail +# Note: if systemd backend is chosen as the default but you enable a jail # for which logs are present only in its own log files, specify some other # backend for that jail (e.g. polling) and provide empty value for # journalmatch. See https://github.com/fail2ban/fail2ban/issues/959#issuecomment-74901200 From a42aa726ab22d07980ba811bf09151c1e49ce2a4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 20:13:03 +0100 Subject: [PATCH 09/15] fixed fail2ban-regex reads invalid character (in sense of given encoding); continuing to process line ignoring invalid characters (still has no test cases). filter test cases added for same issue inside fail2ban-server / fail2ban-testcases; closes gh-1248 --- bin/fail2ban-regex | 10 +++++-- fail2ban/tests/filtertestcase.py | 49 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index b7b0579f..fdbfcb7a 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -84,8 +84,14 @@ def file_lines_gen(hdlr): try: line = line.decode(fail2banRegex.encoding, 'strict') except UnicodeDecodeError: - if sys.version_info >= (3,): # Python 3 must be decoded - line = line.decode(fail2banRegex.encoding, 'ignore') + logSys.warning( + "Error decoding line from '%s' with '%s'." + " Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r" % + ('', fail2banRegex.encoding, line)) + # decode with replacing error chars: + line = line.decode(fail2banRegex.encoding, 'replace') yield line def journal_lines_gen(myjournal): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index acf15528..b15494f5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -90,7 +90,11 @@ def _assert_equal_entries(utest, found, output, count=None): found_time, output_time = \ MyTime.localtime(found[2]),\ MyTime.localtime(output[2]) - utest.assertEqual(found_time, output_time) + try: + utest.assertEqual(found_time, output_time) + except AssertionError as e: + # assert more structured: + utest.assertEqual((float(found[2]), found_time), (float(output[2]), output_time)) if len(output) > 3 and count is None: # match matches # do not check if custom count (e.g. going through them twice) if os.linesep != '\n' or sys.platform.startswith('cygwin'): @@ -216,6 +220,14 @@ class BasicFilter(unittest.TestCase): ("^%Y-%m-%d-%H%M%S.%f %z", "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset")) + def testAssertWrongTime(self): + self.assertRaises(AssertionError, + lambda: _assert_equal_entries(self, + ('1.1.1.1', 1, 1421262060.0), + ('1.1.1.1', 1, 1421262059.0), + 1) + ) + class IgnoreIP(LogCaptureTestCase): @@ -900,6 +912,41 @@ class GetFailures(unittest.TestCase): except FailManagerEmpty: pass + def testGetFailuresWrongChar(self): + # write wrong utf-8 char: + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + fout = fopen(fname, 'wb') + try: + # write: + for l in ( + b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char + b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars + b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars + ): + fout.write(l) + fout.close() + # + output = ('192.0.2.0', 3, 1421262060.0) + failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" + + # encoding - auto + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + # test direct set of encoding: + for enc in ('utf-8', 'ascii'): + self.tearDown();self.setUp(); + self.filter.setLogEncoding('utf-8'); + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + finally: + _killfile(fout, fname) + def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ('93.184.216.34', 2, 1124013539.0, From 46b116e86abb37428220972dfb064d0a8d101a67 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 21:52:06 +0100 Subject: [PATCH 10/15] filter test cases improved + log captured inside such tests + python 3.x compatibility; changelog entry; --- ChangeLog | 2 ++ fail2ban/tests/filtertestcase.py | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..862f46f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) * Fixed default banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (gh-1216) + * Fixed `fail2ban-regex` stops working on invalid (wrong encoded) character + for python version < 3.x (gh-1248) - New Features: * New filters: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b15494f5..3674a574 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -822,7 +822,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover return MonitorJournalFailures -class GetFailures(unittest.TestCase): +class GetFailures(LogCaptureTestCase): FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") @@ -837,6 +837,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) @@ -848,6 +849,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" tearDownMyTime() + LogCaptureTestCase.tearDown(self) def testTail(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) @@ -929,20 +931,20 @@ class GetFailures(unittest.TestCase): output = ('192.0.2.0', 3, 1421262060.0) failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" - # encoding - auto - self.filter.addLogPath(fname) - self.filter.addFailRegex(failregex) - self.filter.getFailures(fname) - _assert_correct_last_attempt(self, self.filter, output) - - # test direct set of encoding: - for enc in ('utf-8', 'ascii'): - self.tearDown();self.setUp(); - self.filter.setLogEncoding('utf-8'); + # test encoding auto or direct set of encoding: + for enc in (None, 'utf-8', 'ascii'): + if enc is not None: + self.tearDown();self.setUp(); + self.filter.setLogEncoding(enc); + self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); finally: _killfile(fout, fname) From 0877d662287bb74cb403c0159841dbbafda28c08 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:46:19 +0100 Subject: [PATCH 11/15] fail2ban-regex moved to the client + test cases for initial coverage added --- .../client/fail2banregex.py | 203 +++++++++--------- fail2ban/server/filter.py | 18 +- fail2ban/tests/fail2banregextestcase.py | 155 +++++++++++++ fail2ban/tests/files/testcase-wrong-char.log | 4 + fail2ban/tests/utils.py | 4 + 5 files changed, 275 insertions(+), 109 deletions(-) rename bin/fail2ban-regex => fail2ban/client/fail2banregex.py (81%) create mode 100644 fail2ban/tests/fail2banregextestcase.py create mode 100644 fail2ban/tests/files/testcase-wrong-char.log diff --git a/bin/fail2ban-regex b/fail2ban/client/fail2banregex.py similarity index 81% rename from bin/fail2ban-regex rename to fail2ban/client/fail2banregex.py index fdbfcb7a..12dde7f1 100755 --- a/bin/fail2ban-regex +++ b/fail2ban/client/fail2banregex.py @@ -44,16 +44,16 @@ from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderErro try: from systemd import journal - from fail2ban.server.filtersystemd import FilterSystemd + from ..server.filtersystemd import FilterSystemd except ImportError: journal = None -from fail2ban.version import version -from fail2ban.client.filterreader import FilterReader -from fail2ban.server.filter import Filter -from fail2ban.server.failregex import RegexException +from ..version import version +from .filterreader import FilterReader +from ..server.filter import Filter, FileContainer +from ..server.failregex import RegexException -from fail2ban.helpers import FormatterWithTraceBack, getLogger +from ..helpers import FormatterWithTraceBack, getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") @@ -63,6 +63,9 @@ def debuggexURL(sample, regex): 'flavor': 'python' }) return 'http://www.debuggex.com/?' + q +def output(args): + print(args) + def shortstr(s, l=53): """Return shortened string """ @@ -77,22 +80,7 @@ def pprint_list(l, header=None): s = "|- %s\n" % header else: s = '' - print s + "| " + "\n| ".join(l) + '\n`-' - -def file_lines_gen(hdlr): - for line in hdlr: - try: - line = line.decode(fail2banRegex.encoding, 'strict') - except UnicodeDecodeError: - logSys.warning( - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r" % - ('', fail2banRegex.encoding, line)) - # decode with replacing error chars: - line = line.decode(fail2banRegex.encoding, 'replace') - yield line + output( s + "| " + "\n| ".join(l) + '\n`-' ) def journal_lines_gen(myjournal): while True: @@ -259,14 +247,14 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - print "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) + output( "Use datepattern : %s" % ( + self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: self._filter.setMaxLines(int(v)) self._maxlines_set = True - print "Use maxlines : %d" % self._filter.getMaxLines() + output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): if self._journalmatch is None: @@ -280,18 +268,18 @@ class Fail2banRegex(object): ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): basedir = os.path.dirname(os.path.dirname(value)) value = os.path.splitext(os.path.basename(value))[0] - print "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) + output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir) if not reader.read(): - print "ERROR: failed to load filter %s" % value + output( "ERROR: failed to load filter %s" % value ) return False else: ## foreign file - readexplicit this file and includes if possible: - print "Use %11s file : %s" % (regex, value) + output( "Use %11s file : %s" % (regex, value) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config) reader.setBaseDir(None) if not reader.readexplicit(): - print "ERROR: failed to read %s" % value + output( "ERROR: failed to read %s" % value ) return False reader.getOptions(None) readercommands = reader.convert() @@ -307,8 +295,8 @@ class Fail2banRegex(object): try: self.setMaxLines(maxlines) except ValueError: - print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ - "read from %(value)s" % locals() + output( "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() ) return False elif command[2] == 'addjournalmatch': journalmatch = command[3:] @@ -317,7 +305,7 @@ class Fail2banRegex(object): datepattern = command[3] self.setDatePattern(datepattern) else: - print "Use %11s line : %s" % (regex, shortstr(value)) + output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = [RegexStat(value)] setattr(self, "_" + regex, regex_values) @@ -335,7 +323,7 @@ class Fail2banRegex(object): found = True regex = self._ignoreregex[ret].inc() except RegexException, e: - print e + output( e ) return False return found @@ -352,10 +340,10 @@ class Fail2banRegex(object): regex.inc() regex.appendIP(match) except RegexException, e: - print e + output( e ) return False except IndexError: - print "Sorry, but no found in regex" + output( "Sorry, but no found in regex" ) return False for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -376,7 +364,7 @@ class Fail2banRegex(object): t0 = time.time() for line_no, line in enumerate(test_lines): if isinstance(line, tuple): - line_datetimestripped, ret = fail2banRegex.testRegex( + line_datetimestripped, ret = self.testRegex( line[0], line[1]) line = "".join(line[0]) else: @@ -384,8 +372,8 @@ class Fail2banRegex(object): if line.startswith('#') or not line: # skip comment and empty lines continue - line_datetimestripped, ret = fail2banRegex.testRegex(line) - is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped) + line_datetimestripped, ret = self.testRegex(line) + is_ignored = self.testIgnoreRegex(line_datetimestripped) if is_ignored: self._line_stats.ignored += 1 @@ -432,18 +420,18 @@ class Fail2banRegex(object): b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) elif lines < self._maxlines or getattr(self, '_print_all_' + ltype): pprint_list([x.rstrip() for x in l], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) def printStats(self): - print - print "Results" - print "=======" + output( "" ) + output( "Results" ) + output( "=======" ) def print_failregexes(title, failregexes): # Print title @@ -464,7 +452,7 @@ class Fail2banRegex(object): timeString, ip[-1] and " (multiple regex matched)" or "")) - print "\n%s: %d total" % (title, total) + output( "\n%s: %d total" % (title, total) ) pprint_list(out, " #) [# of hits] regular expression") return total @@ -474,7 +462,7 @@ class Fail2banRegex(object): if self._filter.dateDetector is not None: - print "\nDate template hits:" + output( "\nDate template hits:" ) out = [] for template in self._filter.dateDetector.templates: if self._verbose or template.hits: @@ -482,10 +470,10 @@ class Fail2banRegex(object): template.hits, template.name)) pprint_list(out, "[# of hits] date format") - print "\nLines: %s" % self._line_stats, + output( "\nLines: %s" % self._line_stats, ) if self._time_elapsed is not None: - print "[processed in %.2f sec]" % self._time_elapsed, - print + output( "[processed in %.2f sec]" % self._time_elapsed, ) + output( "" ) if self._print_all_matched: self.printLines('matched') @@ -496,9 +484,62 @@ class Fail2banRegex(object): return True + def file_lines_gen(self, hdlr): + for line in hdlr: + yield FileContainer.decode_line('', self.encoding, line) -if __name__ == "__main__": + def start(self, opts, args): + cmd_log, cmd_regex = args[:2] + + if not self.readRegex(cmd_regex, 'fail'): + return False + + if len(args) == 3 and not self.readRegex(args[2], 'ignore'): + return False + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log, 'rb') + output( "Use log file : %s" % cmd_log ) + output( "Use encoding : %s" % self.encoding ) + test_lines = self.file_lines_gen(hdlr) + except IOError, e: + output( e ) + return False + elif cmd_log == "systemd-journal": + if not journal: + output( "Error: systemd library not found. Exiting..." ) + return False + myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + journalmatch = self._journalmatch + self.setDatePattern(None) + if journalmatch: + try: + for element in journalmatch: + if element == "+": + myjournal.add_disjunction() + else: + myjournal.add_match(element) + except ValueError: + output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) + return False + output( "Use journal match : %s" % " ".join(journalmatch) ) + test_lines = journal_lines_gen(myjournal) + else: + output( "Use single line : %s" % shortstr(cmd_log) ) + test_lines = [ cmd_log ] + output( "" ) + + self.process(test_lines) + + if not self.printStats(): + return False + + return True + + +def exec_command_line(): # pragma: no cover parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -510,18 +551,16 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) - print - print "Running tests" - print "=============" - print - - fail2banRegex = Fail2banRegex(opts) - # We need 2 or 3 parameters if not len(args) in (2, 3): sys.stderr.write("ERROR: provide both and .\n\n") parser.print_help() - sys.exit(-1) + return False + + output( "" ) + output( "Running tests" ) + output( "=============" ) + output( "" ) # TODO: taken from -testcases -- move common functionality somewhere if opts.log_level is not None: # pragma: no cover @@ -552,46 +591,6 @@ if __name__ == "__main__": stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) - cmd_log, cmd_regex = args[:2] - - fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) - - if len(args) == 3: - fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) - - if os.path.isfile(cmd_log): - try: - hdlr = open(cmd_log, 'rb') - print "Use log file : %s" % cmd_log - print "Use encoding : %s" % fail2banRegex.encoding - test_lines = file_lines_gen(hdlr) - except IOError, e: - print e - sys.exit(-1) - elif cmd_log == "systemd-journal": - if not journal: - print "Error: systemd library not found. Exiting..." - sys.exit(-1) - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) - journalmatch = fail2banRegex._journalmatch - fail2banRegex.setDatePattern(None) - if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - print "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) - sys.exit(-1) - print "Use journal match : %s" % " ".join(journalmatch) - test_lines = journal_lines_gen(myjournal) - else: - print "Use single line : %s" % shortstr(cmd_log) - test_lines = [ cmd_log ] - print - - fail2banRegex.process(test_lines) - - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex = Fail2banRegex(opts) + if not fail2banRegex.start(opts, args): + sys.exit(-1) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6fc2cd6c..65be0467 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -792,23 +792,27 @@ class FileContainer: self.__handler.seek(self.__pos) return True - def readline(self): - if self.__handler is None: - return "" - line = self.__handler.readline() + @staticmethod + def decode_line(filename, enc, line): try: - line = line.decode(self.getEncoding(), 'strict') + line = line.decode(enc, 'strict') except UnicodeDecodeError: logSys.warning( "Error decoding line from '%s' with '%s'." " Consider setting logencoding=utf-8 (or another appropriate" " encoding) for this jail. Continuing" " to process line ignoring invalid characters: %r" % - (self.getFileName(), self.getEncoding(), line)) + (filename, enc, line)) # decode with replacing error chars: - line = line.decode(self.getEncoding(), 'replace') + line = line.decode(enc, 'replace') return line + def readline(self): + if self.__handler is None: + return "" + return FileContainer.decode_line( + self.getFileName(), self.getEncoding(), self.__handler.readline()) + def close(self): if not self.__handler is None: # Saves the last position. diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py new file mode 100644 index 00000000..ee10128e --- /dev/null +++ b/fail2ban/tests/fail2banregextestcase.py @@ -0,0 +1,155 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__author__ = "Serg Brester" +__copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors" +__license__ = "GPL" + +from __builtin__ import open as fopen +import unittest +import getpass +import os +import sys +import time +import tempfile +import uuid + +try: + from systemd import journal +except ImportError: + journal = None + +from ..client import fail2banregex +from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output +from .utils import LogCaptureTestCase, logSys + + +fail2banregex.logSys = logSys +def _test_output(*args): + logSys.info(args[0]) + +fail2banregex.output = _test_output + +CONF_FILES_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__),"..", "..", "config")) +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + + +def _Fail2banRegex(*args): + parser = get_opt_parser() + (opts, args) = parser.parse_args(list(args)) + return (opts, args, Fail2banRegex(opts)) + +class Fail2banRegexTest(LogCaptureTestCase): + + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") + FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") + FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") + + FILTER_SSHD = os.path.join(CONF_FILES_DIR, 'filter.d', 'sshd.conf') + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testWrongRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".** from $" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testWrongIngnoreRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".*? from $", r".**" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testDirectFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", "--print-no-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + + def testDirectNotFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"XYZ from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed') + + def testDirectIgnored(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-ignored", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $", + r"kevin from 192.0.2.0$" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed') + + def testDirectRE_1(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_01, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters') + + self.assertLogged('Dez 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128') + self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10') + + def testDirectRE_2(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_02, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + def testWronChar(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); + + self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') + self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') diff --git a/fail2ban/tests/files/testcase-wrong-char.log b/fail2ban/tests/files/testcase-wrong-char.log new file mode 100644 index 00000000..9736020e --- /dev/null +++ b/fail2ban/tests/files/testcase-wrong-char.log @@ -0,0 +1,4 @@ +Nov 8 00:16:12 main sshd[32547]: Invalid user llinco\361ir from 192.0.2.0 +Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco\361ir +Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llincoñir +Nov 8 00:16:14 main sshd[32547]: Failed password for invalid user llinco\361ir from 192.0.2.0 port 57025 ssh2 diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 35fa59fd..8172e7ec 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -85,6 +85,7 @@ def gatherTests(regexps=None, no_network=False): from . import misctestcase from . import databasetestcase from . import samplestestcase + from . import fail2banregextestcase if not regexps: # pragma: no cover tests = unittest.TestSuite() @@ -152,6 +153,9 @@ def gatherTests(regexps=None, no_network=False): # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) + # bin/fail2ban-regex + tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest)) + # # Python action testcases # From 38f09b417ad514254b81edb210322d6658c5a9e9 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:48:37 +0100 Subject: [PATCH 12/15] fail2ban-regex command line (after fail2ban-regex functionality moved to the client) --- bin/fail2ban-regex | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 bin/fail2ban-regex diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex new file mode 100755 index 00000000..584c1ea7 --- /dev/null +++ b/bin/fail2ban-regex @@ -0,0 +1,34 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" +__license__ = "GPL" + +from fail2ban.client.fail2banregex import exec_command_line + +exec_command_line() From 689dfa1e6a27aa49e0c06895d7cb049e6352174c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 12:19:46 +0100 Subject: [PATCH 13/15] debuggexURL fixed for wrong encoded character; test cases extended; --- fail2ban/client/fail2banregex.py | 15 ++++++++----- fail2ban/tests/fail2banregextestcase.py | 30 +++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 12dde7f1..92ad4a91 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -204,7 +204,7 @@ class LineStats(object): # just for convenient str def __getitem__(self, key): - return getattr(self, key) + return getattr(self, key) if hasattr(self, key) else '' class Fail2banRegex(object): @@ -240,7 +240,11 @@ class Fail2banRegex(object): else: self.encoding = locale.getpreferredencoding() + def decode_line(self, line): + return FileContainer.decode_line('', self.encoding, line) + def encode_line(self, line): + return line.encode(self.encoding, 'ignore') def setDatePattern(self, pattern): if not self._datepattern_set: @@ -398,8 +402,6 @@ class Fail2banRegex(object): self._filter.dateDetector.sortTemplate() self._time_elapsed = time.time() - t0 - - def printLines(self, ltype): lstats = self._line_stats assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) @@ -417,7 +419,8 @@ class Fail2banRegex(object): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) + b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ @@ -486,7 +489,7 @@ class Fail2banRegex(object): def file_lines_gen(self, hdlr): for line in hdlr: - yield FileContainer.decode_line('', self.encoding, line) + yield self.decode_line(line) def start(self, opts, args): @@ -507,7 +510,7 @@ class Fail2banRegex(object): except IOError, e: output( e ) return False - elif cmd_log == "systemd-journal": + elif cmd_log == "systemd-journal": # pragma: no cover if not journal: output( "Error: systemd library not found. Exiting..." ) return False diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index ee10128e..2fd362c7 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -60,6 +60,8 @@ def _Fail2banRegex(*args): class Fail2banRegexTest(LogCaptureTestCase): + RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") @@ -120,7 +122,7 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_01, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') @@ -135,11 +137,23 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_02, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + def testVerbose(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--verbose", "--print-no-missed", + Fail2banRegexTest.FILENAME_02, + Fail2banRegexTest.RE_00 + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + self.assertLogged('141.3.81.106 Fri Aug 14 11:53:59 2015') + self.assertLogged('141.3.81.106 Fri Aug 14 11:54:59 2015') + def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD @@ -153,3 +167,15 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') + + def testWronCharDebuggex(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--debuggex", "--print-all-matched", + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('http://') + + From 441dffbe2a539a3910c5ae4fc4e1341d9a103640 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:31:56 -0500 Subject: [PATCH 14/15] ENH: Pruned some "pragma: no cover"s in fail2banregex This code should and can be unit-tested, so no reason to keep it with no cover --- fail2ban/client/fail2banregex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 92ad4a91..d0cdab84 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -542,7 +542,7 @@ class Fail2banRegex(object): return True -def exec_command_line(): # pragma: no cover +def exec_command_line(): parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -566,10 +566,10 @@ def exec_command_line(): # pragma: no cover output( "" ) # TODO: taken from -testcases -- move common functionality somewhere - if opts.log_level is not None: # pragma: no cover + if opts.log_level is not None: # so we had explicit settings logSys.setLevel(getattr(logging, opts.log_level.upper())) - else: # pragma: no cover + else: # suppress the logging but it would leave unittests' progress dots # ticking, unless like with '-l critical' which would be silent # unless error occurs @@ -587,9 +587,9 @@ def exec_command_line(): # pragma: no cover Formatter = logging.Formatter # Custom log format for the verbose tests runs - if opts.verbose: # pragma: no cover + if opts.verbose: stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) - else: # pragma: no cover + else: # just prefix with the space stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) From b3ed19b36adb17bfbd550bd0e32aac34e9ab8243 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:47:13 -0500 Subject: [PATCH 15/15] DOC: tune up to jail.conf.5 - some line breaks, typos etc --- man/jail.conf.5 | 50 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 7a31e8b1..865c689e 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -1,4 +1,4 @@ -.TH JAIL.CONF "5" "October 2013" "Fail2Ban" "Fail2Ban Configuration" +.TH JAIL.CONF "5" "November 2015" "Fail2Ban" "Fail2Ban Configuration" .SH NAME jail.conf \- configuration for the fail2ban server .SH SYNOPSIS @@ -89,8 +89,8 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. -Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g. .RS .nf @@ -100,7 +100,7 @@ failregex = %(known/failregex)s .fi .RE -Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file, e.g. .RS # filter.d/test.conf: @@ -118,7 +118,7 @@ filter = test[test.method=POST, baduseragents="badagent|"] .fi .RE -Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. +Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X, '; ' can only be used on the first line due to an Python library bug. .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" @@ -130,34 +130,44 @@ The items that can be set are: verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG. Default: ERROR .TP .B logtarget -log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR . Only a single log target can be specified. +log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR +.br +Only a single log target can be specified. If you change logtarget from the default value and you are using logrotate -- also adjust or disable rotation in the corresponding configuration file (e.g. /etc/logrotate.d/fail2ban on Debian systems). .TP .B socket -socket filename. Default: /var/run/fail2ban/fail2ban.sock . +socket filename. Default: /var/run/fail2ban/fail2ban.sock +.br This is used for communication with the fail2ban server daemon. Do not remove this file when Fail2ban is running. It will not be possible to communicate with the server afterwards. .TP .B pidfile -PID filename. Default: /var/run/fail2ban/fail2ban.pid. +PID filename. Default: /var/run/fail2ban/fail2ban.pid +.br This is used to store the process ID of the fail2ban server. .TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 +.br This defines where the persistent data for fail2ban is stored. This persistent data allows bans to be reinstated and continue reading log files from the last read position when fail2ban is restarted. A value of \fINone\fR disables this feature. .TP .B dbpurgeage Database purge age in seconds. Default: 86400 (24hours) +.br This sets the age at which bans should be purged from the database. .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP .B filter -name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. Only one filter can be specified. +name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. +.br +Only one filter can be specified. .TP .B logpath -filename(s) of the log files to be monitored, separated by new lines. Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. +filename(s) of the log files to be monitored, separated by new lines. +.br +Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. Optional space separated option 'tail' can be added to the end of the path to cause the log file to be read from the end, else default 'head' option reads file from the beginning @@ -167,13 +177,17 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. +.br +This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action -action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: +action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. +.br +Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS .RS @@ -187,7 +201,9 @@ Values can also be quoted (required when value includes a ","). More that one ac list of IPs not to ban. They can include a CIDR mask too. .TP .B ignorecommand -command that is executed to determine if the current candidate IP for banning should not be banned. IP will not be banned if command returns successfully (exit code 0). +command that is executed to determine if the current candidate IP for banning should not be banned. +.br +IP will not be banned if command returns successfully (exit code 0). Like ACTION FILES, tags like are can be included in the ignorecommand value and will be substituted before execution. Currently only is supported however more will be added later. .TP .B bantime @@ -200,7 +216,9 @@ time interval (in seconds) before the current time where failures will count tow number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP. .TP .B backend -backend to be used to detect changes in the logpath. It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. +backend to be used to detect changes in the logpath. +.br +It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. .TP .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. @@ -271,9 +289,9 @@ The maximum period of time in seconds that a command can executed, before being .RE Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should -return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). +return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed.