From a544c5abac03dbc124708afbcd1912cbd9f2686a Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 8 Jul 2016 12:22:38 +0200 Subject: [PATCH 01/91] sshd-filter: recognized "Failed publickey for" now (gh-1477) + improved regexp (not anchored now to recognize all "Failed anything for ... from " ChangeLog entry added --- ChangeLog | 3 +++ config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c4444e2a..f441b99b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,9 @@ releases. - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) * `filter.d/postfix-sasl.conf` - Allow for having no trailing space after 'failed:' (gh-1497) +* filter.d/sshd.conf + - recognized "Failed publickey for" (gh-1477); + - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index eeb1518e..7003d1b2 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$ + ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(?:: | (?!from )|$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7baf4be7..a6b07222 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -118,6 +118,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } +Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI # failJSON: { "match": false } Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth] @@ -161,4 +163,3 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root - From 2c54f9046970046265f6f6f5d18cdca19e331940 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 18 Aug 2016 21:34:09 +0200 Subject: [PATCH 02/91] sshd-filter: better universal regexp, that matches more complex different injects, using conditional expressions (on username and auth-info section), see new test cases also. --- ChangeLog | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f441b99b..2bc060ee 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,7 @@ releases. * filter.d/sshd.conf - recognized "Failed publickey for" (gh-1477); - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) + - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 7003d1b2..9a3d40f0 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(?:: | (?!from )|$) + ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index a6b07222..c53bbadb 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -117,6 +117,10 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2004-11-11T08:04:52", "match": true , "host": "127.0.0.1", "desc": "More complex injecting on username ssh 'test from 10.10.1.2 port 55555 ssh2'@localhost" } +Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from 10.10.1.2 port 55555 ssh2 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2004-11-11T08:04:52", "match": true , "host": "127.0.0.1", "desc": "More complex injecting on auth-info ssh test@localhost, auth-info: ' from 10.10.1.2 port 55555 ssh2'" } +Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from 127.0.0.1 port 58946 ssh2: from 10.10.1.2 port 55555 ssh2 # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI From 84c3eb3e0ee08058d23cc81b7954813287dd29f8 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 15 Oct 2016 14:53:08 +0200 Subject: [PATCH 03/91] filter.d/sendmail-reject.conf: double space (should be by missing dns-host only) Closes #1578 --- config/filter.d/sendmail-reject.conf | 2 +- fail2ban/tests/files/logs/sendmail-reject | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index 93b8343c..20d3648e 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -23,7 +23,7 @@ _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) failregex = ^%(__prefix_line)s\w{14}: ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[\]( \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ ^%(__prefix_line)sruleset=check_relay, arg1=(?P\S+), arg2=, relay=((?P=dom) )?\[(\d+\.){3}\d+\]( \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^%(__prefix_line)s\w{14}: rejecting commands from (\S+ )?\[\] due to pre-greeting traffic after \d+ seconds$ + ^%(__prefix_line)s\w{14}: rejecting commands from (\S* )?\[\] due to pre-greeting traffic after \d+ seconds$ ^%(__prefix_line)s\w{14}: (\S+ )?\[\]: ((?i)expn|vrfy) \S+ \[rejected\]$ ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[\]$ diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index b326cf43..70d4dde6 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -40,6 +40,8 @@ Feb 19 18:01:50 batman sm-mta[78152]: ruleset=check_relay, arg1=[196.213.73.146] # failJSON: { "time": "2005-02-27T10:53:06", "match": true , "host": "209.15.212.253" } Feb 27 10:53:06 batman sm-mta[44307]: s1R9r60D044307: rejecting commands from [209.15.212.253] due to pre-greeting traffic after 0 seconds +# failJSON: { "time": "2005-02-27T10:53:07", "match": true , "host": "1.2.3.4" } +Feb 27 10:53:07 strange sm-mta[18001]: u9A0GtpL018001: rejecting commands from example.com [1.2.3.4] due to pre-greeting traffic after 6 seconds # failJSON: { "time": "2005-02-27T15:44:18", "match": true , "host": "41.204.78.137" } Feb 27 15:44:18 batman sm-mta[87838]: s1REiHdq087838: ruleset=check_rcpt, arg1=, relay=[41.204.78.137], reject=550 5.7.1 ... Relaying denied. IP name lookup failed [41.204.78.137] From 519e355bf2cb524775dfae4087c577f9292cf9a1 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 15 Oct 2016 14:59:36 +0200 Subject: [PATCH 04/91] ChangeLog entry added --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 57737828..2a6b5733 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,8 @@ releases. - Allow for having no trailing space after 'failed:' (gh-1497) * `filter.d/vsftpd.conf` - Optional reason part in message after FAIL LOGIN (gh-1543) +* `filter.d/sendmail-reject.conf` + - removed mandatory double space (if dns-host available, gh-1579) ### New Features From 7805f9972d361eeb9ded6eabd69d84a7618c0fe7 Mon Sep 17 00:00:00 2001 From: Aaron Lindsay Date: Sat, 15 Oct 2016 12:02:45 -0400 Subject: [PATCH 05/91] filter.d/sshd.conf: Match 'Invalid user' with 'port \d*' --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index eeb1518e..9fff72ac 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -22,7 +22,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ - ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ + ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from (?: port \d*)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7baf4be7..be5896cc 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -162,3 +162,6 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root +# Match invalid user messages with port at the end +# failJSON: {"time": "2004-10-15T11:35:28", "match": true , "host": "1.2.3.4", "desc": "Invalid user root" } +Oct 15 11:35:28 somehost sshd[7024]: Invalid user root from 1.2.3.4 port 37220 From a7d9de8c5264b20f1fa63071f3f4eeed8735b54a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 27 Sep 2016 17:53:45 +0200 Subject: [PATCH 06/91] [temp commit] 1st try to optimize datedetector/datetemplate functionality (fix ambiguous resp. misleading date detection if several formats used in log resp. by format switch after restart of some services): * Misleading date patterns defined more precisely (using extended syntax %E[mdHMS] for exact two-digit match) * `filter.d/freeswitch.conf` - Optional prefixes (server, daemon, dual time) if systemd daemon logs used (gh-1548) - User part rewritten to accept IPv6 resp. domain after "@" (gh-1548) --- config/filter.d/freeswitch.conf | 15 +++- fail2ban/server/datedetector.py | 99 +++++++++++++++------- fail2ban/server/datetemplate.py | 32 +++---- fail2ban/server/strptime.py | 16 ++++ fail2ban/tests/datedetectortestcase.py | 112 ++++++++++++++----------- fail2ban/tests/files/logs/freeswitch | 5 ++ 6 files changed, 181 insertions(+), 98 deletions(-) diff --git a/config/filter.d/freeswitch.conf b/config/filter.d/freeswitch.conf index 1ef5a256..29940240 100644 --- a/config/filter.d/freeswitch.conf +++ b/config/filter.d/freeswitch.conf @@ -8,10 +8,21 @@ # IP addresses on your LAN. # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] -failregex = ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[.*\] from ip $ - ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ Can't find user \[\d+@\d+\.\d+\.\d+\.\d+\] from $ +_daemon = freeswitch + +# Prefix contains common prefix line (server, daemon, etc.) and 2 datetimes if used systemd backend +_pref_line = ^%(__prefix_line)s(?:\d+-\d+-\d+ \d+:\d+:\d+\.\d+)? + +failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip $ + %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ Can't find user \[[^@]+@[^\]]+\] from $ ignoreregex = diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 5281fc59..b1f97af7 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -21,6 +21,7 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import copy import time from threading import Lock @@ -35,6 +36,8 @@ logLevel = 6 class DateDetectorCache(object): + """Implements the caching of the default templates list. + """ def __init__(self): self.__lock = Lock() self.__templates = list() @@ -51,26 +54,44 @@ class DateDetectorCache(object): def _cacheTemplate(self, template): """Cache Fail2Ban's default template. + """ if isinstance(template, str): + # exact given template with word benin-end boundary: template = DatePatternRegex(template) + # additional template, that prefers datetime at start of a line (safety+performance feature): + template2 = copy.copy(template) + if hasattr(template, 'pattern'): + regex = template.pattern + wordEnd = True + else: + regex = template.regex + wordEnd = False + template2.setRegex(regex, wordBegin='start', wordEnd=wordEnd) + if template2.name != template.name: + self.__templates.append(template2) + # add template: self.__templates.append(template) def _addDefaultTemplate(self): """Add resp. cache Fail2Ban's default set of date templates. """ + # ISO 8601, simple date, optional subsecond and timezone: + # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59 + # simple date: 2005/01/23 21:59:59 + # custom for syslog-ng 2006.12.21 06:43:20 + self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") + # 20050123T215959, 20050123 215959 + self._cacheTemplate("%Y%Em%Ed[T ]%EH%EM%ES(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # prefixed with optional time zone (monit): + # PDT Apr 16 21:05:29 + self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") - # simple date, optional subsecond (proftpd): - # 2005-01-23 21:59:59 - # simple date: 2005/01/23 21:59:59 - # custom for syslog-ng 2006.12.21 06:43:20 - self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?") # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) @@ -79,37 +100,48 @@ class DateDetectorCache(object): # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] # 26-Jul-2007 15:20:52 + # named 26-Jul-2007 15:20:52.252 + # roundcube 26-Jul-2007 15:20:52 +0200 self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") # CPanel 05/20/2008:01:57:39 self._cacheTemplate("%m/%d/%Y:%H:%M:%S") - # named 26-Jul-2007 15:20:52.252 - # roundcube 26-Jul-2007 15:20:52 +0200 # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion - # with previous - self._cacheTemplate("%m-%d-%Y %H:%M:%S\.%f") + # with previous ("%d-%m-%Y %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") + self._cacheTemplate("%m-%d-%Y %H:%M:%S(?:\.%f)?") # TAI64N - template = DateTai64n() - template.name = "TAI64N" - self._cacheTemplate(template) + self._cacheTemplate(DateTai64n()) # Epoch - template = DateEpoch() - template.name = "Epoch" - self._cacheTemplate(template) - # ISO 8601 - self._cacheTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") + self._cacheTemplate(DateEpoch()) # Only time information in the log self._cacheTemplate("^%H:%M:%S") # <09/16/08@05:03:30> self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>") # MySQL: 130322 11:46:11 - self._cacheTemplate("^%y%m%d ?%H:%M:%S") + self._cacheTemplate("%y%Em%Ed ?%H:%M:%S") # Apache Tomcat self._cacheTemplate("%b %d, %Y %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 self._cacheTemplate("^%b-%d-%y %H:%M:%S") +class DateDetectorTemplate(object): + """Used for "shallow copy" of the template object. + + Prevents collectively usage of hits/lastUsed in cached templates + """ + __slots__ = ('template', 'hits', 'lastUsed') + def __init__(self, template): + self.template = template + self.hits = 0 + self.lastUsed = 0 + + def __getattr__(self, name): + """ Returns attribute of template (called for parameters not in slots) + """ + return getattr(self.template, name) + + class DateDetector(object): """Manages one or more date templates to find a date within a log line. @@ -132,7 +164,7 @@ class DateDetector(object): raise ValueError( "There is already a template with name %s" % name) self.__known_names.add(name) - self.__templates.append(template) + self.__templates.append(DateDetectorTemplate(template)) def appendTemplate(self, template): """Add a date template to manage and use in search of dates. @@ -186,13 +218,14 @@ class DateDetector(object): """ i = 0 with self.__lock: - for template in self.__templates: + for ddtemplate in self.__templates: + template = ddtemplate.template match = template.matchDate(line) if not match is None: if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, "Matched time template %s", template.name) - template.hits += 1 - template.lastUsed = time.time() + ddtemplate.hits += 1 + ddtemplate.lastUsed = time.time() # if not first - try to reorder current template (bubble up), they will be not sorted anymore: if i: self._reorderTemplate(i) @@ -234,7 +267,8 @@ class DateDetector(object): except ValueError: return None with self.__lock: - for template in self.__templates: + for ddtemplate in self.__templates: + template = ddtemplate.template try: date = template.getDate(line) if date is None: @@ -261,14 +295,21 @@ class DateDetector(object): ## current hits and time the template was long unused: untime = template.lastUsed - self.__unusedTime hits = template.hits + ## try to move faster (first 2 if it still unused, or half of part to current template position): + phits = 0 + for pos in (0, 1, num // 2): + phits = templates[pos].hits + if not phits: + break ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : - if hits > templates[num-1].hits + 5 or templates[num-1].lastUsed < untime: - ## try to move faster (half of part to current template): - pos = num // 2 - ## if not larger - move slow (exact 1 position): - if hits <= templates[pos].hits or templates[pos].lastUsed < untime: + if not phits or hits > phits + 5 or templates[pos].lastUsed < untime: + ## if not larger (and target position recently used) - move slow (exact 1 position): + if hits <= phits and templates[pos].lastUsed > untime: pos = num-1 + ## if still smaller and template at position used, don't move: + if hits < templates[pos].hits and templates[pos].lastUsed > untime: + return templates[pos], templates[num] = template, templates[pos] diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 4f609e78..5fcc16a4 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -27,7 +27,7 @@ __license__ = "GPL" import re from abc import abstractmethod -from .strptime import reGroupDictStrptime, timeRE +from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE from ..helpers import getLogger logSys = getLogger(__name__) @@ -49,8 +49,6 @@ class DateTemplate(object): self._name = "" self._regex = "" self._cRegex = None - self.hits = 0 - self.lastUsed = 0 @property def name(self): @@ -88,9 +86,11 @@ class DateTemplate(object): """ regex = regex.strip() if wordBegin and not re.search(r'^\^', regex): - regex = r'(?=^|\b|\W)' + regex + regex = (r'(?<=^|\b)' if wordBegin != 'start' else r"^(?<=\W)?") + regex + self._name = ('[*WD-BEG]' if wordBegin != 'start' else '[^LN-BEG]') + self._name if wordEnd and not re.search(r'\$$', regex): regex += r'(?=\b|\W|$)' + self._name += ('[*WD-END]' if wordEnd else '') self._regex = regex regex = property(getRegex, setRegex, doc= @@ -140,7 +140,9 @@ class DateEpoch(DateTemplate): def __init__(self): DateTemplate.__init__(self) - self.regex = r"(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" + self.name = "Epoch" + self.setRegex(r"(?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))", + wordBegin=False) ;# already line begin resp. word begin anchored def getDate(self, line, dateMatch=None): """Method to return the date for a log line. @@ -178,7 +180,8 @@ class DatePatternRegex(DateTemplate): regex pattern """ - _patternRE = re.compile(r"%%(%%|[%s])" % "".join(timeRE.keys())) + + _patternRE = re.compile(getTimePatternRE()) _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", 'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month", @@ -188,11 +191,11 @@ class DatePatternRegex(DateTemplate): for _key in set(timeRE) - set(_patternName): # may not have them all... _patternName[_key] = "%%%s" % _key - def __init__(self, pattern=None): + def __init__(self, pattern=None, **kwargs): super(DatePatternRegex, self).__init__() self._pattern = None if pattern is not None: - self.pattern = pattern + self.setRegex(pattern, **kwargs) @property def pattern(self): @@ -208,17 +211,13 @@ class DatePatternRegex(DateTemplate): @pattern.setter def pattern(self, pattern): + self.setRegex(pattern) + + def setRegex(self, pattern, wordBegin=True, wordEnd=True): self._pattern = pattern fmt = self._patternRE.sub(r'%(\1)s', pattern) self._name = fmt % self._patternName - super(DatePatternRegex, self).setRegex(fmt % timeRE) - - def setRegex(self, value): - raise NotImplementedError("Regex derived from pattern") - - @DateTemplate.name.setter - def name(self, value): - raise NotImplementedError("Name derived from pattern") + super(DatePatternRegex, self).setRegex(fmt % timeRE, wordBegin, wordEnd) def getDate(self, line, dateMatch=None): """Method to return the date for a log line. @@ -258,6 +257,7 @@ class DateTai64n(DateTemplate): def __init__(self): DateTemplate.__init__(self) + self.name = "TAI64N" # We already know the format for TAI64N # yoh: we should not add an additional front anchor self.setRegex("@[0-9a-f]{24}", wordBegin=False) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 2e3c051c..c6a1f37e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -26,8 +26,24 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() +#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): +#timeRE['z'] = r"%s?(?PZ|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" +# Extend build-in TimeRE with some exact (two-digit) patterns: +timeRE['Ed'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" +timeRE['Em'] = r"(?P1[0-2]|0[1-9])" +timeRE['EH'] = r"(?P2[0-3]|[0-1]\d)" +timeRE['EM'] = r"(?P[0-5]\d)" +timeRE['ES'] = r"(?P6[0-1]|[0-5]\d)" + +def getTimePatternRE(): + keys = timeRE.keys() + return (r"%%(%%|%s|[%s])" % ( + "|".join([k for k in keys if len(k) > 1]), + "".join([k for k in keys if len(k) == 1]), + )) + def reGroupDictStrptime(found_dict): """Return time from dictionary of strptime fields diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 0c3c306d..013e1e85 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -89,69 +89,79 @@ class DateDetectorTest(LogCaptureTestCase): """ dateUnix = 1106513999.0 - for anchored, sdate in ( - (False, "Jan 23 21:59:59"), - (False, "Sun Jan 23 21:59:59 2005"), - (False, "Sun Jan 23 21:59:59"), - (False, "Sun Jan 23 2005 21:59:59"), - (False, "2005/01/23 21:59:59"), - (False, "2005.01.23 21:59:59"), - (False, "23/01/2005 21:59:59"), - (False, "23/01/05 21:59:59"), - (False, "23/Jan/2005:21:59:59"), - (False, "23/Jan/2005:21:59:59 +0100"), - (False, "01/23/2005:21:59:59"), - (False, "2005-01-23 21:59:59"), - (False, "2005-01-23 21:59:59,000"), # proftpd - (False, "23-Jan-2005 21:59:59"), - (False, "23-Jan-2005 21:59:59.02"), - (False, "23-Jan-2005 21:59:59 +0100"), - (False, "23-01-2005 21:59:59"), - (True, "1106513999"), # Portsetry - (False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break - (False, "@4000000041f4104f00000000"), # TAI64N - (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC) - (False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ - (False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local - (True, "<01/23/05@21:59:59>"), - (True, "050123 21:59:59"), # MySQL - (True, "Jan-23-05 21:59:59"), # ASSP like - (False, "Jan 23, 2005 9:59:59 PM"), # Apache Tomcat - (True, "1106513999"), # Regular epoch - (True, "1106513999.000"), # Regular epoch with millisec - (False, "audit(1106513999.000:987)"), # SELinux + for anchored, bound, sdate, rdate in ( + (False, True, "Jan 23 21:59:59", None), + (False, False, "Sun Jan 23 21:59:59 2005", None), + (False, False, "Sun Jan 23 21:59:59", None), + (False, False, "Sun Jan 23 2005 21:59:59", None), + (False, True, "2005/01/23 21:59:59", None), + (False, True, "2005.01.23 21:59:59", None), + (False, True, "23/01/2005 21:59:59", None), + (False, True, "23/01/05 21:59:59", None), + (False, True, "23/Jan/2005:21:59:59", None), + (False, True, "23/Jan/2005:21:59:59 +0100", None), + (False, True, "01/23/2005:21:59:59", None), + (False, True, "2005-01-23 21:59:59", None), + (False, True, "2005-01-23 21:59:59,000", None), # proftpd + (False, True, "23-Jan-2005 21:59:59", None), + (False, True, "23-Jan-2005 21:59:59.02", None), + (False, True, "23-Jan-2005 21:59:59 +0100", None), + (False, True, "23-01-2005 21:59:59", None), + (True, True, "1106513999", None), # Portsetry + (False, True, "01-23-2005 21:59:59.252", None), # reported on f2b, causes Feb29 fix to break + (False, False, "@4000000041f4104f00000000", None), # TAI64N + (False, True, "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC) + (False, True, "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ + (False, True, "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local + (False, True, "20050123T215959", None), #Short ISO + (False, True, "20050123 215959", None), #Short ISO + (True, True, "<01/23/05@21:59:59>", None), + (False, True, "050123 21:59:59", None), # MySQL + (True, True, "Jan-23-05 21:59:59", None), # ASSP like + (False, True, "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat + (True, True, "1106513999", None), # Regular epoch + (True, True, "1106513999.000", None), # Regular epoch with millisec + (True, True, "[1106513999.000]", "1106513999.000"), # epoch squared + (False, True, "audit(1106513999.000:987)", "1106513999.000"), # SELinux + ): + logSys.debug('== test %r', (anchored, bound, sdate)) + for should_match, prefix in ( + (True, ""), + (not anchored, "bogus-prefix "), + (False, "word-boundary") ): - for should_match, prefix in ((True, ""), - (not anchored, "bogus-prefix ")): + if rdate is None: rdate = sdate log = prefix + sdate + "[sshd] error: PAM: Authentication failure" - + # if not allowed boundary test: + if not bound and prefix == "word-boundary": continue + logSys.debug(' -- test %-5s for %r', should_match, log) # with getTime: logtime = self.__datedetector.getTime(log) if should_match: - self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log)) + self.assertNotEqual(logtime, None, + "getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log)) ( logUnix, logMatch ) = logtime - self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix)) - if sdate.startswith('audit('): - # yes, special case, the group only matches the number - self.assertEqual(logMatch.group(), '1106513999.000') - else: - self.assertEqual(logMatch.group(), sdate) + self.assertEqual(logUnix, dateUnix, + "getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) + self.assertEqual(logMatch.group(), rdate) else: - self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) + self.assertEqual(logtime, None, + "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) # with getTime(matchTime) - this combination used in filter: - matchTime = self.__datedetector.matchTime(log) + (timeMatch, template) = matchTime = self.__datedetector.matchTime(log) logtime = self.__datedetector.getTime(log, matchTime) + logSys.debug(' -- found - %r', template.name if timeMatch else False) if should_match: - self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log)) + self.assertNotEqual(logtime, None, + "getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log)) ( logUnix, logMatch ) = logtime - self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix)) - if sdate.startswith('audit('): - # yes, special case, the group only matches the number - self.assertEqual(logMatch.group(), '1106513999.000') - else: - self.assertEqual(logMatch.group(), sdate) + self.assertEqual(logUnix, dateUnix, + "getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) + self.assertEqual(logMatch.group(), rdate) else: - self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) + self.assertEqual(logtime, None, + "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) + logSys.debug(' -- OK') def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.__datedetector.appendTemplate, diff --git a/fail2ban/tests/files/logs/freeswitch b/fail2ban/tests/files/logs/freeswitch index f151995c..2579e6f4 100644 --- a/fail2ban/tests/files/logs/freeswitch +++ b/fail2ban/tests/files/logs/freeswitch @@ -9,3 +9,8 @@ 2013-12-31 17:39:54.767815 [WARNING] sofia_reg.c:2531 Can't find user [1001@192.168.2.51] from 5.11.47.236 # failJSON: { "time": "2013-12-31T17:39:54", "match": true, "host": "185.24.234.141" } 2013-12-31 17:39:54.767815 [WARNING] sofia_reg.c:2531 Can't find user [100@192.168.2.51] from 185.24.234.141 + +# failJSON: { "time": "2016-09-25T18:57:58", "match": true, "host": "192.0.2.1", "desc": "Systemd dual time with prefix - 1st expr" } +2016-09-25T18:57:58.150982 www.srv.tld freeswitch[122921]: 2016-09-25 18:57:58.150982 [WARNING] sofia_reg.c:2889 Can't find user [201@::1] from 192.0.2.1 +# failJSON: { "time": "2016-09-25T18:57:58", "match": true, "host": "192.0.2.2", "desc": "Systemd dual time with prefix - 2nd expr" } +2016-09-25T18:57:58.150982 www.srv.tld freeswitch[122921]: 2016-09-25 18:57:58.150982 [WARNING] sofia_reg.c:1720 SIP auth failure (INVITE) on sofia profile 'sipinterface_1' for [9810972597751739@::1] from ip 192.0.2.2 \ No newline at end of file From 84fe55b99b8010c1951fea021a886a6cbbc4de7d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 28 Sep 2016 21:17:42 +0200 Subject: [PATCH 07/91] [temp commit] 2nd try to optimize datedetector/datetemplate functionality (almost ready, needs fine tuning) --- fail2ban/client/fail2banregex.py | 40 +++-- fail2ban/server/datedetector.py | 105 ++++++------- fail2ban/server/datetemplate.py | 45 +++--- fail2ban/server/mytime.py | 20 ++- fail2ban/server/strptime.py | 51 +++++- fail2ban/tests/datedetectortestcase.py | 147 +++++++++++++++++- fail2ban/tests/fail2banregextestcase.py | 2 +- fail2ban/tests/files/logs/zzz-generic-example | 8 +- fail2ban/tests/filtertestcase.py | 6 +- fail2ban/tests/misctestcase.py | 89 ----------- fail2ban/tests/servertestcase.py | 2 +- fail2ban/tests/utils.py | 10 +- 12 files changed, 305 insertions(+), 220 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 4096ac4e..6680b097 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -122,15 +122,15 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues p.add_options([ Option("-d", "--datepattern", help="set custom pattern used to match date/times"), - Option("-e", "--encoding", + Option("-e", "--encoding", default=PREFER_ENC, help="File encoding. Default: system locale"), - Option("-r", "--raw", action='store_true', + Option("-r", "--raw", action='store_true', default=False, help="Raw hosts, don't resolve dns"), Option("--usedns", action='store', default=None, help="DNS specified replacement of tags in regexp " "('yes' - matches all form of hosts, 'no' - IP addresses only)"), Option("-L", "--maxlines", type=int, default=0, - help="maxlines for multi-line regex"), + help="maxlines for multi-line regex."), Option("-m", "--journalmatch", help="journalctl style matches overriding filter file. " "\"systemd-journal\" only"), @@ -143,6 +143,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues help="Increase verbosity"), Option("--verbosity", action="store", dest="verbose", type=int, help="Set numerical level of verbosity (0..4)"), + Option("--verbose-date", "--VD", action='store_true', + help="Verbose date patterns/regex in output"), Option("-D", "--debuggex", action='store_true', help="Produce debuggex.com urls for debugging there"), Option("--print-no-missed", action='store_true', @@ -215,14 +217,8 @@ class LineStats(object): class Fail2banRegex(object): def __init__(self, opts): - self._verbose = opts.verbose - self._debuggex = opts.debuggex - self._maxlines = 20 - self._print_no_missed = opts.print_no_missed - self._print_no_ignored = opts.print_no_ignored - self._print_all_matched = opts.print_all_matched - self._print_all_missed = opts.print_all_missed - self._print_all_ignored = opts.print_all_ignored + # set local protected memebers from given options: + self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False self._journalmatch = None @@ -236,23 +232,20 @@ class Fail2banRegex(object): if opts.maxlines: self.setMaxLines(opts.maxlines) + else: + self._maxlines = 20 if opts.journalmatch is not None: self.setJournalMatch(opts.journalmatch.split()) if opts.datepattern: self.setDatePattern(opts.datepattern) - if opts.encoding: - self.encoding = opts.encoding - else: - self.encoding = PREFER_ENC - self.raw = True if opts.raw else False if opts.usedns: self._filter.setUseDns(opts.usedns) def decode_line(self, line): - return FileContainer.decode_line('', self.encoding, line) + return FileContainer.decode_line('', self._encoding, line) def encode_line(self, line): - return line.encode(self.encoding, 'ignore') + return line.encode(self._encoding, 'ignore') def setDatePattern(self, pattern): if not self._datepattern_set: @@ -350,7 +343,7 @@ class Fail2banRegex(object): orgLineBuffer = self._filter._Filter__lineBuffer fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() try: - line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw) + line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self._raw) for match in ret: # Append True/False flag depending if line was matched by # more than one regex @@ -479,8 +472,11 @@ class Fail2banRegex(object): out = [] for template in self._filter.dateDetector.templates: if self._verbose or template.hits: - out.append("[%d] %s" % ( - template.hits, template.name)) + out.append("[%d] %s" % (template.hits, template.name)) + if self._verbose_date: + out.append(" # weight: %3s, pattern: %s" % ( + template.weight, getattr(template, 'pattern', ''),)) + out.append(" # regex: %s" % (getattr(template, 'regex', ''),)) pprint_list(out, "[# of hits] date format") output( "\nLines: %s" % self._line_stats, ) @@ -518,7 +514,7 @@ class Fail2banRegex(object): try: hdlr = open(cmd_log, 'rb') output( "Use log file : %s" % cmd_log ) - output( "Use encoding : %s" % self.encoding ) + output( "Use encoding : %s" % self._encoding ) test_lines = self.file_lines_gen(hdlr) except IOError as e: output( e ) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index b1f97af7..d26743a3 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -60,16 +60,15 @@ class DateDetectorCache(object): # exact given template with word benin-end boundary: template = DatePatternRegex(template) # additional template, that prefers datetime at start of a line (safety+performance feature): - template2 = copy.copy(template) - if hasattr(template, 'pattern'): - regex = template.pattern - wordEnd = True - else: - regex = template.regex - wordEnd = False - template2.setRegex(regex, wordBegin='start', wordEnd=wordEnd) - if template2.name != template.name: - self.__templates.append(template2) + if 0 and hasattr(template, 'regex'): + template2 = copy.copy(template) + regex = getattr(template, 'pattern', template.regex) + template2.setRegex(regex, wordBegin='start', wordEnd=True) + if template2.name != template.name: + # increase weight of such templates, because they should be always + # preferred in template sorting process (bubble up): + template2.weight = 100 + self.__templates.append(template2) # add template: self.__templates.append(template) @@ -80,35 +79,35 @@ class DateDetectorCache(object): # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59 # simple date: 2005/01/23 21:59:59 # custom for syslog-ng 2006.12.21 06:43:20 - self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") + self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") # 20050123T215959, 20050123 215959 - self._cacheTemplate("%Y%Em%Ed[T ]%EH%EM%ES(?:[.,]%f)?(?:\s*%z)?") + self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 # prefixed with optional time zone (monit): # PDT Apr 16 21:05:29 - self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 - self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") + self._cacheTemplate("(?:%a )?%b %d %ExY %H:%M:%S(?:\.%f)?") # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) # 17-07-2008 17:23:25 - self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") + self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] # 26-Jul-2007 15:20:52 # named 26-Jul-2007 15:20:52.252 # roundcube 26-Jul-2007 15:20:52 +0200 - self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") + self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") # CPanel 05/20/2008:01:57:39 - self._cacheTemplate("%m/%d/%Y:%H:%M:%S") + self._cacheTemplate("%m/%d/%ExY:%H:%M:%S") # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion - # with previous ("%d-%m-%Y %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") - self._cacheTemplate("%m-%d-%Y %H:%M:%S(?:\.%f)?") + # with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") + self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?") # TAI64N self._cacheTemplate(DateTai64n()) # Epoch @@ -116,13 +115,13 @@ class DateDetectorCache(object): # Only time information in the log self._cacheTemplate("^%H:%M:%S") # <09/16/08@05:03:30> - self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>") + self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>") # MySQL: 130322 11:46:11 - self._cacheTemplate("%y%Em%Ed ?%H:%M:%S") + self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S") # Apache Tomcat - self._cacheTemplate("%b %d, %Y %I:%M:%S %p") + self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 - self._cacheTemplate("^%b-%d-%y %H:%M:%S") + self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") class DateDetectorTemplate(object): @@ -218,14 +217,14 @@ class DateDetector(object): """ i = 0 with self.__lock: - for ddtemplate in self.__templates: - template = ddtemplate.template + for ddtempl in self.__templates: + template = ddtempl.template match = template.matchDate(line) - if not match is None: + if match is not None: if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, "Matched time template %s", template.name) - ddtemplate.hits += 1 - ddtemplate.lastUsed = time.time() + ddtempl.hits += 1 + ddtempl.lastUsed = time.time() # if not first - try to reorder current template (bubble up), they will be not sorted anymore: if i: self._reorderTemplate(i) @@ -254,32 +253,21 @@ class DateDetector(object): The Unix timestamp returned from the first successfully matched template or None if not found. """ - if timeMatch: - template = timeMatch[1] - if template is not None: - try: - date = template.getDate(line, timeMatch[0]) - if date is not None: - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Got time %f for %r using template %s", - date[0], date[1].group(), template.name) - return date - except ValueError: - return None - with self.__lock: - for ddtemplate in self.__templates: - template = ddtemplate.template - try: - date = template.getDate(line) - if date is None: - continue + # search match for all specified templates: + if timeMatch is None: + timeMatch = self.matchTime(line) + # convert: + template = timeMatch[1] + if template is not None: + try: + date = template.getDate(line, timeMatch[0]) + if date is not None: if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Got time %f for %r using template %s", + logSys.log(logLevel, "Got time %f for %r using template %s", date[0], date[1].group(), template.name) return date - except ValueError: # pragma: no cover - pass - return None + except ValueError: + return None def _reorderTemplate(self, num): """Reorder template (bubble up) in template list if hits grows enough. @@ -291,16 +279,16 @@ class DateDetector(object): """ if num: templates = self.__templates - template = templates[num] + ddtempl = templates[num] ## current hits and time the template was long unused: - untime = template.lastUsed - self.__unusedTime - hits = template.hits + untime = ddtempl.lastUsed - self.__unusedTime + hits = ddtempl.hits * ddtempl.template.weight ## try to move faster (first 2 if it still unused, or half of part to current template position): - phits = 0 for pos in (0, 1, num // 2): phits = templates[pos].hits - if not phits: + if not phits: # if we've found an unused break + phits *= templates[pos].template.weight ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : if not phits or hits > phits + 5 or templates[pos].lastUsed < untime: @@ -308,8 +296,9 @@ class DateDetector(object): if hits <= phits and templates[pos].lastUsed > untime: pos = num-1 ## if still smaller and template at position used, don't move: - if hits < templates[pos].hits and templates[pos].lastUsed > untime: + phits = templates[pos].hits * templates[pos].template.weight + if hits < phits and templates[pos].lastUsed > untime: return - templates[pos], templates[num] = template, templates[pos] + templates[pos], templates[num] = ddtempl, templates[pos] diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 5fcc16a4..6548e5ee 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -32,6 +32,9 @@ from ..helpers import getLogger logSys = getLogger(__name__) +RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)') +RE_NO_WRD_BOUND_END = re.compile(r'(? 1 else "".join(exprset) + #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): #timeRE['z'] = r"%s?(?PZ|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" -# Extend build-in TimeRE with some exact (two-digit) patterns: -timeRE['Ed'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" -timeRE['Em'] = r"(?P1[0-2]|0[1-9])" -timeRE['EH'] = r"(?P2[0-3]|[0-1]\d)" -timeRE['EM'] = r"(?P[0-5]\d)" -timeRE['ES'] = r"(?P6[0-1]|[0-5]\d)" +# Extend build-in TimeRE with some exact patterns +# exact two-digit patterns: +timeRE['Exd'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" +timeRE['Exm'] = r"(?P1[0-2]|0[1-9])" +timeRE['ExH'] = r"(?P2[0-3]|[0-1]\d)" +timeRE['ExM'] = r"(?P[0-5]\d)" +timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" +# more precise year patterns, within same century of last year and +# the next 3 years (for possible long uptime of fail2ban); thereby +# respect possible run in the test-cases (alternate date used there): +timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) +timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) +# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates: +timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))" def getTimePatternRE(): keys = timeRE.keys() - return (r"%%(%%|%s|[%s])" % ( + patt = (r"%%(%%|%s|[%s])" % ( "|".join([k for k in keys if len(k) > 1]), "".join([k for k in keys if len(k) == 1]), )) - + names = { + 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", + 'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month", + 'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek", + 'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%", + 'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name", + 'ExLB': '{^LN-BEG}', + } + for key in set(keys) - set(names): # may not have them all... + if key.startswith('Ex'): + kn = names.get(key[2:]) + if kn: + names[key] = "Ex" + kn + continue + names[key] = "%%%s" % key + return (patt, names) def reGroupDictStrptime(found_dict): """Return time from dictionary of strptime fields diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 013e1e85..109a275f 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -30,7 +30,7 @@ import datetime from ..server.datedetector import DateDetector from ..server import datedetector -from ..server.datetemplate import DateTemplate +from ..server.datetemplate import DatePatternRegex, DateTemplate from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase from ..helpers import getLogger @@ -89,6 +89,10 @@ class DateDetectorTest(LogCaptureTestCase): """ dateUnix = 1106513999.0 + # anchored - matching expression (pattern) is anchored + # bound - pattern can be tested using word boundary (e.g. False if contains in front some optional part) + # sdate - date string used in test log-line + # rdate - if specified, the result match, which differs from sdate for anchored, bound, sdate, rdate in ( (False, True, "Jan 23 21:59:59", None), (False, False, "Sun Jan 23 21:59:59 2005", None), @@ -113,15 +117,15 @@ class DateDetectorTest(LogCaptureTestCase): (False, True, "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC) (False, True, "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ (False, True, "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local - (False, True, "20050123T215959", None), #Short ISO - (False, True, "20050123 215959", None), #Short ISO + (False, True, "20050123T215959", None), #Short ISO with T + (False, True, "20050123 215959", None), #Short ISO with space (True, True, "<01/23/05@21:59:59>", None), (False, True, "050123 21:59:59", None), # MySQL (True, True, "Jan-23-05 21:59:59", None), # ASSP like (False, True, "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat (True, True, "1106513999", None), # Regular epoch (True, True, "1106513999.000", None), # Regular epoch with millisec - (True, True, "[1106513999.000]", "1106513999.000"), # epoch squared + (True, True, "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match) (False, True, "audit(1106513999.000:987)", "1106513999.000"), # SELinux ): logSys.debug('== test %r', (anchored, bound, sdate)) @@ -195,6 +199,141 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(t.matchDate('aaaac').group(), 'aaaac') +iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") + +class CustomDateFormatsTest(unittest.TestCase): + + def testIso8601(self): + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00Z")[0]) + self.assertEqual( + date, + datetime.datetime(2007, 1, 25, 12, 0)) + self.assertRaises(TypeError, iso8601.getDate, None) + self.assertRaises(TypeError, iso8601.getDate, date) + + self.assertEqual(iso8601.getDate(""), None) + self.assertEqual(iso8601.getDate("Z"), None) + + self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) + self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+0400")[0]) + self.assertEqual( + date, + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) + self.assertEqual( + date, + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-0400")[0]) + self.assertEqual( + date, + datetime.datetime(2007, 1, 25, 16, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-04")[0]) + self.assertEqual( + date, + datetime.datetime(2007, 1, 25, 16, 0)) + + def testAmbiguousDatePattern(self): + defDD = DateDetector() + defDD.addDefaultTemplate() + for (matched, dp, line) in ( + # positive case: + ('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'), + # ambiguous "unbound" patterns (missed): + (False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'), + (False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'), + # ambiguous "no optional year" patterns (matched): + ('Aug 8 11:25:50', None, 'Aug 8 11:25:50 20030f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 20030f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 20030f2329b8 Authentication failed from 192.0.2.1'), + # direct specified patterns: + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'), + # direct specified patterns (begin/end, missed): + (False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'), + (False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'), + # direct specified unbound patterns (no begin/end boundary): + ('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '192.0.2.1x20:00:00 01.02.2003'), + ('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '20:00:00 01.02.2003x192.0.2.1'), + # pattern enclosed with stars (in comparison to example above): + ('*20:00:00 01.02.2003*', r'\**%H:%M:%S %d.%m.%Y\**', 'test*20:00:00 01.02.2003*test'), + # direct specified patterns (begin/end, matched): + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'), + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'), + # wrong year in 1st date, so failed by convert using not precise year (filter used last known date), + # in the 2nd and 3th tests (with precise year) it should find correct the 2nd date: + (None, r'%Y-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), + ('2003-12-30 00:00:00', r'%ExY-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), + ('2003-12-30 00:00:00', None, "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), + # wrong date recognized short month/day (unbounded date pattern without separator between parts), + # in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date: + ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), + ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), + ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), + # Explicit bound in start of the line using %ExLB key, + # (negative) in the 1st case without line begin boundary - wrong date may be found, + # (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found) + # (positive) and in 3th case with line begin boundary - find the correct date + ("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), + (None, "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), + ("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), + # Explicit bound in start of the line using %ExLB key, + # up to 2 non-alphanumeric chars front, ** - no word boundary on the right + ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), + ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), + ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), + ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), + ): + logSys.debug('== test: %r', (matched, dp, line)) + if dp is None: + dd = defDD + else: + dp = DatePatternRegex(dp) + dd = DateDetector() + dd.appendTemplate(dp) + date = dd.getTime(line) + if matched: + self.assertTrue(date) + self.assertEqual(matched, date[1].group()) + else: + self.assertEqual(date, None) + + # def testAmbiguousUsingOrderedTemplates(self): + # defDD = DateDetector() + # defDD.addDefaultTemplate() + # for (matched, dp, line) in ( + # # wrong date recognized short month/day (unbounded date pattern without separator), + # # in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date: + # ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), + # ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), + # ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), + # ): + # logSys.debug('== test: %r', (matched, dp, line)) + # if dp is None: + # dd = defDD + # else: + # dp = DatePatternRegex(dp) + # dd = DateDetector() + # dd.appendTemplate(dp) + # date = dd.getTime(line) + # if matched: + # self.assertTrue(date) + # self.assertEqual(matched, date[1].group()) + # else: + # self.assertEqual(date, None) + + # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") # self.__datedetector.setDefaultPattern("%b %d %H:%M:%S") diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 4445fe6b..c62377a9 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -178,7 +178,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testVerbose(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--verbose", "--print-no-missed", + "--verbose", "--verbose-date", "--print-no-missed", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 ) diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index 2044c387..51d3974c 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } +# -- wrong time direct in journal-line (using precise year pattern): +# failJSON: { "match": false} 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } +# -- wrong time direct in journal-line (using precise year pattern): +# failJSON: { "match": false} 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" } diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2b57ce47..6c1a637c 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -283,10 +283,10 @@ class BasicFilter(unittest.TestCase): def testGetSetDatePattern(self): self.assertEqual(self.filter.getDatePattern(), (None, "Default Detectors")) - self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z") + self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z **") self.assertEqual(self.filter.getDatePattern(), - ("^%Y-%m-%d-%H%M%S.%f %z", - "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset")) + ("^%Y-%m-%d-%H%M%S.%f %z **", + "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **")) def testAssertWrongTime(self): self.assertRaises(AssertionError, diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 450904d5..908e4f6c 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -23,13 +23,11 @@ __license__ = "GPL" import logging import os -import re import sys import unittest import tempfile import shutil import fnmatch -import datetime from glob import glob from StringIO import StringIO @@ -37,8 +35,6 @@ from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, uni_decode from ..helpers import splitwords -from ..server.datedetector import DateDetector -from ..server.datetemplate import DatePatternRegex from ..server.mytime import MyTime @@ -320,91 +316,6 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertRaisesRegexp(Exception, 'not all arguments converted', lambda: logSys.debug('test', 1, 2, 3)) -iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") - - -class CustomDateFormatsTest(unittest.TestCase): - - def testIso8601(self): - date = datetime.datetime.utcfromtimestamp( - iso8601.getDate("2007-01-25T12:00:00Z")[0]) - self.assertEqual( - date, - datetime.datetime(2007, 1, 25, 12, 0)) - self.assertRaises(TypeError, iso8601.getDate, None) - self.assertRaises(TypeError, iso8601.getDate, date) - - self.assertEqual(iso8601.getDate(""), None) - self.assertEqual(iso8601.getDate("Z"), None) - - self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) - self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) - date = datetime.datetime.utcfromtimestamp( - iso8601.getDate("2007-01-25T12:00:00+0400")[0]) - self.assertEqual( - date, - datetime.datetime(2007, 1, 25, 8, 0)) - date = datetime.datetime.utcfromtimestamp( - iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) - self.assertEqual( - date, - datetime.datetime(2007, 1, 25, 8, 0)) - date = datetime.datetime.utcfromtimestamp( - iso8601.getDate("2007-01-25T12:00:00-0400")[0]) - self.assertEqual( - date, - datetime.datetime(2007, 1, 25, 16, 0)) - date = datetime.datetime.utcfromtimestamp( - iso8601.getDate("2007-01-25T12:00:00-04")[0]) - self.assertEqual( - date, - datetime.datetime(2007, 1, 25, 16, 0)) - - def testAmbiguousDatePattern(self): - defDD = DateDetector() - defDD.addDefaultTemplate() - logSys = DefLogSys - for (matched, dp, line) in ( - # positive case: - ('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'), - # ambiguous "unbound" patterns (missed): - (False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'), - (False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'), - # ambiguous "no optional year" patterns (matched): - ('Aug 8 11:25:50', None, 'Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1'), - ('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1'), - ('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 14430f2329b8 Authentication failed from 192.0.2.1'), - # direct specified patterns: - ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'), - ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'), - ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), - ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'), - ('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), - ('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'), - ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'), - ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'), - # direct specified patterns (begin/end, missed): - (False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'), - (False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'), - # direct specified patterns (begin/end, matched): - ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'), - ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'), - ): - logSys.debug('== test: %r', (matched, dp, line)) - if dp is None: - dd = defDD - else: - dp = DatePatternRegex(dp) - dd = DateDetector() - dd.appendTemplate(dp) - date = dd.getTime(line) - if matched: - self.assertTrue(date) - self.assertEqual(matched, date[1].group()) - else: - self.assertEqual(date, None) - - class MyTimeTest(unittest.TestCase): def testStr2Seconds(self): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 56c85e94..d4320257 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -298,7 +298,7 @@ class Transmitter(TransmitterBase): def testDatePattern(self): self.setGetTest("datepattern", "%%%Y%m%d%H%M%S", - ("%%%Y%m%d%H%M%S", "%YearMonthDay24hourMinuteSecond"), + ("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"), jail=self.jailName) self.setGetTest( "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index f60dfd1f..22079456 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -48,6 +48,8 @@ from ..version import version logSys = getLogger(__name__) +TEST_NOW = 1124013600 + CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None) if not CONFIG_DIR: @@ -257,6 +259,10 @@ def initTests(opts): def F2B_SkipIfNoNetwork(): raise unittest.SkipTest('Skip test because of "--no-network"') unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork + + # set alternate now for time related test cases: + MyTime.setAlternateNow(TEST_NOW) + # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName for i in xrange(255): @@ -289,7 +295,7 @@ def setUpMyTime(): # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match os.environ['TZ'] = 'Europe/Zurich' time.tzset() - MyTime.setTime(1124013600) + MyTime.setTime(TEST_NOW) def tearDownMyTime(): @@ -384,7 +390,6 @@ def gatherTests(regexps=None, opts=None): tests.addTest(unittest.makeSuite(misctestcase.HelpersTest)) tests.addTest(unittest.makeSuite(misctestcase.SetupTest)) tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest)) - tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest)) tests.addTest(unittest.makeSuite(misctestcase.MyTimeTest)) # Database tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest)) @@ -404,6 +409,7 @@ def gatherTests(regexps=None, opts=None): # DateDetector tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) + tests.addTest(unittest.makeSuite(datedetectortestcase.CustomDateFormatsTest)) # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) From 75a5440acf96c1b3fda37eef9b86de92ee3b1c17 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Sep 2016 20:37:02 +0200 Subject: [PATCH 08/91] extends date detector template with distance (position of match in log-line), to prevent grave collision using (re)ordered template list (e.g. find-spot of wrong date-match inside foreign input, misleading date patterns by ambiguous formats, etc.); By change of the distance (e.g. another format found), the pattern with smallest distance will be always preferred now. To speedup (template lookup) resp. minimize of list reorder counts, the distance will be used as divider factor of the template weight by the templates comparison. --- fail2ban/server/datedetector.py | 81 ++++++++++++++++++-------- fail2ban/tests/datedetectortestcase.py | 47 ++++++++------- 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index d26743a3..63ae3692 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -129,11 +129,13 @@ class DateDetectorTemplate(object): Prevents collectively usage of hits/lastUsed in cached templates """ - __slots__ = ('template', 'hits', 'lastUsed') + __slots__ = ('template', 'hits', 'lastUsed', 'distance') def __init__(self, template): self.template = template self.hits = 0 self.lastUsed = 0 + # the last distance to date-match within the log file: + self.distance = 0x7fffffff def __getattr__(self, name): """ Returns attribute of template (called for parameters not in slots) @@ -156,6 +158,8 @@ class DateDetector(object): self.__known_names = set() # time the template was long unused (currently 300 == 5m): self.__unusedTime = 300 + # first free place: + self.__firstUnused = 0 def _appendTemplate(self, template): name = template.name @@ -215,22 +219,52 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ + match = None i = 0 + found = None, 0x7fffffff, -1 with self.__lock: for ddtempl in self.__templates: template = ddtempl.template match = template.matchDate(line) if match is not None: + distance = max(1, match.start() + 1) if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Matched time template %s", template.name) - ddtempl.hits += 1 - ddtempl.lastUsed = time.time() - # if not first - try to reorder current template (bubble up), they will be not sorted anymore: - if i: - self._reorderTemplate(i) - # return tuple with match and template reference used for parsing: - return (match, template) + logSys.log(logLevel, " matched time template #%r (at %r <= %r) %s", + i, distance, ddtempl.distance, template.name) + ## [grave] if distance changed, possible date-match was found somewhere + ## in body of message, so save this template, and search further: + if distance > ddtempl.distance and len(self.__templates) > 1: + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " ** distance collision - pattern change, reserve") + ## shortest of both: + if distance < found[1]: + found = match, distance, i + ## search further: + match = None + i += 1 + continue + ## winner - stop search: + break i += 1 + # check other template was found (use this one with shortest distance): + if match is None and found[0]: + match, distance, i = found + ddtempl = self.__templates[i] + template = ddtempl.template + # we've winner, incr hits, set distance, usage, reorder, etc: + if match is not None: + ddtempl.hits += 1 + ddtempl.distance = distance + ddtempl.lastUsed = time.time() + if self.__firstUnused == i: + self.__firstUnused += 1 + # if not first - try to reorder current template (bubble up), they will be not sorted anymore: + if i: + logSys.log(logLevel, " -> reorder template #%r, hits: %r", i, ddtempl.hits) + self._reorderTemplate(i) + # return tuple with match and template reference used for parsing: + return (match, template) + # not found: return (None, None) @@ -263,7 +297,7 @@ class DateDetector(object): date = template.getDate(line, timeMatch[0]) if date is not None: if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Got time %f for %r using template %s", + logSys.log(logLevel, " got time %f for %r using template %s", date[0], date[1].group(), template.name) return date except ValueError: @@ -282,23 +316,24 @@ class DateDetector(object): ddtempl = templates[num] ## current hits and time the template was long unused: untime = ddtempl.lastUsed - self.__unusedTime - hits = ddtempl.hits * ddtempl.template.weight - ## try to move faster (first 2 if it still unused, or half of part to current template position): - for pos in (0, 1, num // 2): - phits = templates[pos].hits - if not phits: # if we've found an unused - break - phits *= templates[pos].template.weight + weight = ddtempl.hits * ddtempl.template.weight / ddtempl.distance + ## try to move faster (first if unused available, or half of part to current template position): + pos = self.__firstUnused if self.__firstUnused < num else num // 2 + pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : - if not phits or hits > phits + 5 or templates[pos].lastUsed < untime: + logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight) + if not pweight or weight > pweight + 5 or templates[pos].lastUsed < untime: ## if not larger (and target position recently used) - move slow (exact 1 position): - if hits <= phits and templates[pos].lastUsed > untime: + if weight <= pweight and templates[pos].lastUsed > untime: pos = num-1 ## if still smaller and template at position used, don't move: - phits = templates[pos].hits * templates[pos].template.weight - if hits < phits and templates[pos].lastUsed > untime: + pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance + logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight) + if weight < pweight and templates[pos].lastUsed > untime: return templates[pos], templates[num] = ddtempl, templates[pos] - - + logSys.log(logLevel, " -> moved template #%r -> #%r", num, pos) + ## correct first unused: + if pos == self.__firstUnused: + self.__firstUnused += 1 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 109a275f..9616edbb 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -198,6 +198,29 @@ class DateDetectorTest(LogCaptureTestCase): self.assertRaises(Exception, t.getDate, '') self.assertEqual(t.matchDate('aaaac').group(), 'aaaac') + def testAmbiguousInOrderedTemplates(self): + dd = DateDetector() + dd.addDefaultTemplate() + for (debit, line, cnt) in ( + ("2003-03-07 17:05:01", "some free text 2003-03-07 17:05:01 test ...", 15), + # distance collision detection (date from foreign input should not be found): + ("030324 0:04:00", "server mysqld[1000]: 030324 0:04:00 [Warning] Access denied ..." + " foreign-input just some free text 2003-03-07 17:05:01 test", 10), + # distance collision detection (first date should be found): + ("Sep 16 21:30:26", "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 10), + # just to test sorting: + ("2005-10-07 06:09:42", "server mysqld[5906]: 2005-10-07 06:09:42 5907 [Warning] Access denied", 20), + ("2005-10-08T15:26:18.237955", "server mysqld[5906]: 2005-10-08T15:26:18.237955 6 [Note] Access denied", 20), + # date format changed again: + ("051009 10:05:30", "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 20), + ): + logSys.debug('== test: %r', (debit, line, cnt)) + for i in range(cnt): + logSys.debug('Line: %s', line) + match, template = dd.matchTime(line) + self.assertTrue(match) + self.assertEqual(match.group(), debit) + iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") @@ -309,30 +332,6 @@ class CustomDateFormatsTest(unittest.TestCase): else: self.assertEqual(date, None) - # def testAmbiguousUsingOrderedTemplates(self): - # defDD = DateDetector() - # defDD.addDefaultTemplate() - # for (matched, dp, line) in ( - # # wrong date recognized short month/day (unbounded date pattern without separator), - # # in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date: - # ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), - # ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), - # ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), - # ): - # logSys.debug('== test: %r', (matched, dp, line)) - # if dp is None: - # dd = defDD - # else: - # dp = DatePatternRegex(dp) - # dd = DateDetector() - # dd.appendTemplate(dp) - # date = dd.getTime(line) - # if matched: - # self.assertTrue(date) - # self.assertEqual(matched, date[1].group()) - # else: - # self.assertEqual(date, None) - # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") From b9033d004ed7e4179e82b44d582dafd95b443b67 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Oct 2016 17:25:49 +0200 Subject: [PATCH 09/91] amend distance collision check - always find template with shortest distance, also first time (test-case extended); datedetector property template without lock, initially placed start-anchored templates at begin of template list, small optimization of strptime --- fail2ban/server/datedetector.py | 39 +++++++---- fail2ban/server/datetemplate.py | 8 +-- fail2ban/server/strptime.py | 92 ++++++++++++++------------ fail2ban/tests/datedetectortestcase.py | 3 + 4 files changed, 79 insertions(+), 63 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 63ae3692..e2c14d05 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -46,6 +46,8 @@ class DateDetectorCache(object): def templates(self): """List of template instances managed by the detector. """ + if self.__templates: + return self.__templates with self.__lock: if self.__templates: return self.__templates @@ -60,21 +62,22 @@ class DateDetectorCache(object): # exact given template with word benin-end boundary: template = DatePatternRegex(template) # additional template, that prefers datetime at start of a line (safety+performance feature): - if 0 and hasattr(template, 'regex'): + if hasattr(template, 'regex'): template2 = copy.copy(template) regex = getattr(template, 'pattern', template.regex) template2.setRegex(regex, wordBegin='start', wordEnd=True) if template2.name != template.name: # increase weight of such templates, because they should be always # preferred in template sorting process (bubble up): - template2.weight = 100 - self.__templates.append(template2) + template2.weight = 100.0 + self.__tmpcache[0].append(template2) # add template: - self.__templates.append(template) + self.__tmpcache[1].append(template) def _addDefaultTemplate(self): """Add resp. cache Fail2Ban's default set of date templates. """ + self.__tmpcache = [], [] # ISO 8601, simple date, optional subsecond and timezone: # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59 # simple date: 2005/01/23 21:59:59 @@ -122,6 +125,8 @@ class DateDetectorCache(object): self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") + self.__templates = self.__tmpcache[0] + self.__tmpcache[1] + del self.__tmpcache class DateDetectorTemplate(object): @@ -158,6 +163,8 @@ class DateDetector(object): self.__known_names = set() # time the template was long unused (currently 300 == 5m): self.__unusedTime = 300 + # last known distance: + self.__lastDistance = 0 # first free place: self.__firstUnused = 0 @@ -229,11 +236,14 @@ class DateDetector(object): if match is not None: distance = max(1, match.start() + 1) if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " matched time template #%r (at %r <= %r) %s", - i, distance, ddtempl.distance, template.name) + logSys.log(logLevel, " matched time template #%r (at %r <= %r, %r) %s", + i, distance, ddtempl.distance, self.__lastDistance, template.name) ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: - if distance > ddtempl.distance and len(self.__templates) > 1: + if ( + (distance > ddtempl.distance or distance > self.__lastDistance) and + len(self.__templates) > 1 + ): if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: @@ -254,7 +264,7 @@ class DateDetector(object): # we've winner, incr hits, set distance, usage, reorder, etc: if match is not None: ddtempl.hits += 1 - ddtempl.distance = distance + self.__lastDistance = ddtempl.distance = distance ddtempl.lastUsed = time.time() if self.__firstUnused == i: self.__firstUnused += 1 @@ -322,18 +332,21 @@ class DateDetector(object): pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : - logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight) - if not pweight or weight > pweight + 5 or templates[pos].lastUsed < untime: + logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r, hits %r > %r", + num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) + if not pweight or weight > pweight or templates[pos].lastUsed < untime: ## if not larger (and target position recently used) - move slow (exact 1 position): if weight <= pweight and templates[pos].lastUsed > untime: pos = num-1 ## if still smaller and template at position used, don't move: pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance - logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight) + logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r, hits %r > %r", + num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) if weight < pweight and templates[pos].lastUsed > untime: return - templates[pos], templates[num] = ddtempl, templates[pos] - logSys.log(logLevel, " -> moved template #%r -> #%r", num, pos) + del templates[num] + templates[pos:0] = [ddtempl] ## correct first unused: if pos == self.__firstUnused: self.__firstUnused += 1 + logSys.log(logLevel, " -> moved template #%r -> #%r", num, pos) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 6548e5ee..3d116b1e 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -50,7 +50,7 @@ class DateTemplate(object): def __init__(self): self.name = "" - self.weight = 1 + self.weight = 1.0 self._regex = "" self._cRegex = None @@ -232,11 +232,7 @@ class DatePatternRegex(DateTemplate): if not dateMatch: dateMatch = self.matchDate(line) if dateMatch: - groupdict = dict( - (key, value) - for key, value in dateMatch.groupdict().iteritems() - if value is not None) - return reGroupDictStrptime(groupdict), dateMatch + return reGroupDictStrptime(dateMatch.groupdict()), dateMatch class DateTai64n(DateTemplate): diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 2012aae6..85045ed8 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -80,7 +80,7 @@ def getTimePatternRE(): names[key] = "%%%s" % key return (patt, names) -def reGroupDictStrptime(found_dict): +def reGroupDictStrptime(found_dict, msec=False): """Return time from dictionary of strptime fields This is tweaked from python built-in _strptime. @@ -109,14 +109,15 @@ def reGroupDictStrptime(found_dict): # weekday and julian defaulted to -1 so as to signal need to calculate # values weekday = julian = -1 - for group_key in found_dict.keys(): + for key, val in found_dict.iteritems(): + if val is None: continue # Directives not explicitly handled below: # c, x, X # handled by making out of other directives # U, W # worthless without day of the week - if group_key == 'y': - year = int(found_dict['y']) + if key == 'y': + year = int(val) # Open Group specification for strptime() states that a %y #value in the range of [00, 68] is in the century 2000, while #[69,99] is in the century 1900 @@ -124,20 +125,20 @@ def reGroupDictStrptime(found_dict): year += 2000 else: year += 1900 - elif group_key == 'Y': - year = int(found_dict['Y']) - elif group_key == 'm': - month = int(found_dict['m']) - elif group_key == 'B': - month = locale_time.f_month.index(found_dict['B'].lower()) - elif group_key == 'b': - month = locale_time.a_month.index(found_dict['b'].lower()) - elif group_key == 'd': - day = int(found_dict['d']) - elif group_key == 'H': - hour = int(found_dict['H']) - elif group_key == 'I': - hour = int(found_dict['I']) + elif key == 'Y': + year = int(val) + elif key == 'm': + month = int(val) + elif key == 'B': + month = locale_time.f_month.index(val.lower()) + elif key == 'b': + month = locale_time.a_month.index(val.lower()) + elif key == 'd': + day = int(val) + elif key == 'H': + hour = int(val) + elif key == 'I': + hour = int(val) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM if ampm in ('', locale_time.am_pm[0]): @@ -152,37 +153,38 @@ def reGroupDictStrptime(found_dict): # 12 noon == 12 PM == hour 12 if hour != 12: hour += 12 - elif group_key == 'M': - minute = int(found_dict['M']) - elif group_key == 'S': - second = int(found_dict['S']) - elif group_key == 'f': - s = found_dict['f'] - # Pad to always return microseconds. - s += "0" * (6 - len(s)) - fraction = int(s) - elif group_key == 'A': - weekday = locale_time.f_weekday.index(found_dict['A'].lower()) - elif group_key == 'a': - weekday = locale_time.a_weekday.index(found_dict['a'].lower()) - elif group_key == 'w': - weekday = int(found_dict['w']) + elif key == 'M': + minute = int(val) + elif key == 'S': + second = int(val) + elif key == 'f': + if msec: + s = val + # Pad to always return microseconds. + s += "0" * (6 - len(s)) + fraction = int(s) + elif key == 'A': + weekday = locale_time.f_weekday.index(val.lower()) + elif key == 'a': + weekday = locale_time.a_weekday.index(val.lower()) + elif key == 'w': + weekday = int(val) if weekday == 0: weekday = 6 else: weekday -= 1 - elif group_key == 'j': - julian = int(found_dict['j']) - elif group_key in ('U', 'W'): - week_of_year = int(found_dict[group_key]) - if group_key == 'U': + elif key == 'j': + julian = int(val) + elif key in ('U', 'W'): + week_of_year = int(val) + if key == 'U': # U starts week on Sunday. week_of_year_start = 6 else: # W starts week on Monday. week_of_year_start = 0 - elif group_key == 'z': - z = found_dict['z'] + elif key == 'z': + z = val if z == "Z": tzoffset = 0 else: @@ -227,7 +229,7 @@ def reGroupDictStrptime(found_dict): # Actully create date date_result = datetime.datetime( year, month, day, hour, minute, second, fraction) - if gmtoff: + if gmtoff is not None: date_result = date_result - datetime.timedelta(seconds=gmtoff) if date_result > now and assume_today: @@ -240,7 +242,9 @@ def reGroupDictStrptime(found_dict): year=year-1, month=month, day=day) if gmtoff is not None: - return calendar.timegm(date_result.utctimetuple()) + tm = calendar.timegm(date_result.utctimetuple()) else: - return time.mktime(date_result.timetuple()) - + tm = time.mktime(date_result.timetuple()) + if msec: + tm += fraction/1000000.0 + return tm diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 9616edbb..b862bbdb 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -202,6 +202,9 @@ class DateDetectorTest(LogCaptureTestCase): dd = DateDetector() dd.addDefaultTemplate() for (debit, line, cnt) in ( + # shortest distance to datetime should win: + ("030324 0:03:59", "some free text 030324 0:03:59 -- 2003-03-07 17:05:01 ...", 1), + # some free text with datetime: ("2003-03-07 17:05:01", "some free text 2003-03-07 17:05:01 test ...", 15), # distance collision detection (date from foreign input should not be found): ("030324 0:04:00", "server mysqld[1000]: 030324 0:04:00 [Warning] Access denied ..." From f56ff5f48b1075e59884e86a1a18332d98311c77 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 5 Oct 2016 12:11:06 +0200 Subject: [PATCH 10/91] optimized to better usage of the last time template (parse part of line at the same place as last time, if enclosed in the same boundaries) thereby follow rule "shortest distance to datetime should win", so possible collision causes search though all templates; speedup it a little bit (not possible collision if distance <= 1 or if line-begin anchoring, so break search if such template found) --- fail2ban/server/datedetector.py | 132 +++++++++++++++++-------- fail2ban/server/datetemplate.py | 21 +++- fail2ban/tests/datedetectortestcase.py | 57 +++++++---- 3 files changed, 145 insertions(+), 65 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index e2c14d05..ba2b6377 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -26,7 +26,7 @@ import time from threading import Lock -from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch +from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch from ..helpers import getLogger # Gets the instance of the logger. @@ -142,6 +142,10 @@ class DateDetectorTemplate(object): # the last distance to date-match within the log file: self.distance = 0x7fffffff + @property + def weight(self): + return self.hits * self.template.weight / max(1, self.distance) + def __getattr__(self, name): """ Returns attribute of template (called for parameters not in slots) """ @@ -158,13 +162,14 @@ class DateDetector(object): _defCache = DateDetectorCache() def __init__(self): - self.__lock = Lock() self.__templates = list() self.__known_names = set() # time the template was long unused (currently 300 == 5m): self.__unusedTime = 300 - # last known distance: - self.__lastDistance = 0 + # last known distance (bypass one char collision) and end position: + self.__lastPos = 1, None + self.__lastEndPos = 0x7fffffff, None + self.__lastTemplIdx = 0x7fffffff # first free place: self.__firstUnused = 0 @@ -198,9 +203,8 @@ class DateDetector(object): def addDefaultTemplate(self): """Add Fail2Ban's default set of date templates. """ - with self.__lock: - for template in DateDetector._defCache.templates: - self._appendTemplate(template) + for template in DateDetector._defCache.templates: + self._appendTemplate(template) @property def templates(self): @@ -226,29 +230,59 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ + #logSys.log(logLevel, "try to match time for line: %.250s", line) match = None - i = 0 - found = None, 0x7fffffff, -1 - with self.__lock: + # first try to use last template with same start/end position: + i = self.__lastTemplIdx + if i < len(self.__templates): + ddtempl = self.__templates[i] + template = ddtempl.template + distance, endpos = self.__lastPos[0], self.__lastEndPos[0] + # check same boundaries left/right, otherwise possible collision/pattern switch: + if (line[distance-1:distance] == self.__lastPos[1] and + line[endpos:endpos+1] == self.__lastEndPos[1] + ): + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ... %s ...", + i, distance, endpos, line[distance:endpos]) + match = template.matchDate(line, distance, endpos) + if match: + distance = match.start() + endpos = match.end() + # if different position, possible collision/pattern switch: + if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]: + logSys.log(logLevel, " matched last time template #%02i", i) + else: + logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") + match = None + # search template and better match: + if not match: + self.__lastTemplIdx = 0x7fffffff + logSys.log(logLevel, " search template ...") + found = None, 0x7fffffff, -1 + i = 0 for ddtempl in self.__templates: template = ddtempl.template match = template.matchDate(line) - if match is not None: - distance = max(1, match.start() + 1) + if match: + distance = match.start() + endpos = match.end() if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " matched time template #%r (at %r <= %r, %r) %s", - i, distance, ddtempl.distance, self.__lastDistance, template.name) + logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", + i, distance, ddtempl.distance, self.__lastPos[0], template.name) + ## if line-begin anchored - stop searching: + if template.flags & DateTemplate.LINE_BEGIN: + break ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: if ( - (distance > ddtempl.distance or distance > self.__lastDistance) and + (distance > ddtempl.distance or distance > self.__lastPos[0]) and len(self.__templates) > 1 ): - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " ** distance collision - pattern change, reserve") + logSys.log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: - found = match, distance, i + found = match, distance, endpos, i ## search further: match = None i += 1 @@ -257,25 +291,29 @@ class DateDetector(object): break i += 1 # check other template was found (use this one with shortest distance): - if match is None and found[0]: - match, distance, i = found + if not match and found[0]: + match, distance, endpos, i = found + logSys.log(logLevel, " use best time template #%02i", i) ddtempl = self.__templates[i] template = ddtempl.template - # we've winner, incr hits, set distance, usage, reorder, etc: - if match is not None: - ddtempl.hits += 1 - self.__lastDistance = ddtempl.distance = distance - ddtempl.lastUsed = time.time() - if self.__firstUnused == i: - self.__firstUnused += 1 - # if not first - try to reorder current template (bubble up), they will be not sorted anymore: - if i: - logSys.log(logLevel, " -> reorder template #%r, hits: %r", i, ddtempl.hits) - self._reorderTemplate(i) - # return tuple with match and template reference used for parsing: - return (match, template) + # we've winner, incr hits, set distance, usage, reorder, etc: + if match: + ddtempl.hits += 1 + ddtempl.lastUsed = time.time() + ddtempl.distance = distance + if self.__firstUnused == i: + self.__firstUnused += 1 + self.__lastPos = distance, line[distance-1:distance] + self.__lastEndPos = endpos, line[endpos:endpos+1] + # if not first - try to reorder current template (bubble up), they will be not sorted anymore: + if i: + i = self._reorderTemplate(i) + self.__lastTemplIdx = i + # return tuple with match and template reference used for parsing: + return (match, template) # not found: + logSys.log(logLevel, " no template.") return (None, None) def getTime(self, line, timeMatch=None): @@ -311,7 +349,8 @@ class DateDetector(object): date[0], date[1].group(), template.name) return date except ValueError: - return None + pass + return None def _reorderTemplate(self, num): """Reorder template (bubble up) in template list if hits grows enough. @@ -324,29 +363,36 @@ class DateDetector(object): if num: templates = self.__templates ddtempl = templates[num] + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " -> reorder template #%02i, hits: %r", num, ddtempl.hits) ## current hits and time the template was long unused: untime = ddtempl.lastUsed - self.__unusedTime - weight = ddtempl.hits * ddtempl.template.weight / ddtempl.distance + weight = ddtempl.weight ## try to move faster (first if unused available, or half of part to current template position): pos = self.__firstUnused if self.__firstUnused < num else num // 2 - pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance + pweight = templates[pos].weight ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : - logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r, hits %r > %r", - num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r", + num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) if not pweight or weight > pweight or templates[pos].lastUsed < untime: ## if not larger (and target position recently used) - move slow (exact 1 position): if weight <= pweight and templates[pos].lastUsed > untime: pos = num-1 ## if still smaller and template at position used, don't move: - pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance - logSys.log(logLevel, " -> compare template #%r & #%r, weight %r > %r, hits %r > %r", - num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) + pweight = templates[pos].weight + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r", + num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) if weight < pweight and templates[pos].lastUsed > untime: return del templates[num] templates[pos:0] = [ddtempl] ## correct first unused: - if pos == self.__firstUnused: + while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits: self.__firstUnused += 1 - logSys.log(logLevel, " -> moved template #%r -> #%r", num, pos) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " -> moved template #%02i -> #%02i", num, pos) + return pos + return num diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 3d116b1e..d85c8f66 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -48,9 +48,14 @@ class DateTemplate(object): regex """ + LINE_BEGIN = 8 + WORD_BEGIN = 2 + WORD_END = 1 + def __init__(self): self.name = "" self.weight = 1.0 + self.flags = 0 self._regex = "" self._cRegex = None @@ -83,12 +88,14 @@ class DateTemplate(object): regex = regex.strip() # if word or line start boundary: if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex): + self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN regex = (r'(?=^|\b|\W)' if wordBegin != 'start' else r"(?:^|(?<=^\W)|(?<=^\W{2}))") + regex self.name = ('{*WD-BEG}' if wordBegin != 'start' else '{^LN-BEG}') + self.name # if word end boundary: if wordEnd and not RE_NO_WRD_BOUND_END.search(regex): + self.flags |= DateTemplate.WORD_END regex += r'(?=\b|\W|$)' - self.name += ('{*WD-END}' if wordEnd else '') + self.name += '{*WD-END}' # remove possible special pattern "**" in front and end of regex: regex = RE_DEL_WRD_BOUNDS.sub('', regex) self._regex = regex @@ -97,12 +104,18 @@ class DateTemplate(object): """Regex used to search for date. """) - def matchDate(self, line): - """Check if regex for date matches on a log line. + def _compileRegex(self): + """Compile regex by first usage. """ if not self._cRegex: self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE) - dateMatch = self._cRegex.search(line) + + def matchDate(self, line, *args): + """Check if regex for date matches on a log line. + """ + if not self._cRegex: + self._compileRegex() + dateMatch = self._cRegex.search(line, *args); # pos, endpos return dateMatch @abstractmethod diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index b862bbdb..a4c9b645 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -42,17 +42,20 @@ class DateDetectorTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" LogCaptureTestCase.setUp(self) - self.__old_eff_level = datedetector.logLevel - datedetector.logLevel = logSys.getEffectiveLevel() setUpMyTime() - self.__datedetector = DateDetector() - self.__datedetector.addDefaultTemplate() + self.__datedetector = None def tearDown(self): """Call after every test case.""" LogCaptureTestCase.tearDown(self) - datedetector.logLevel = self.__old_eff_level tearDownMyTime() + + @property + def datedetector(self): + if self.__datedetector is None: + self.__datedetector = DateDetector() + self.__datedetector.addDefaultTemplate() + return self.__datedetector def testGetEpochTime(self): # correct epoch time, using all variants: @@ -60,7 +63,7 @@ class DateDetectorTest(LogCaptureTestCase): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): date = date % dateUnix log = date + " [sshd] error: PAM: Authentication failure" - datelog = self.__datedetector.getTime(log) + datelog = self.datedetector.getTime(log) self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) @@ -70,7 +73,7 @@ class DateDetectorTest(LogCaptureTestCase): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): date = date % dateUnix log = date + " [sshd] error: PAM: Authentication failure" - datelog = self.__datedetector.getTime(log) + datelog = self.datedetector.getTime(log) self.assertFalse(datelog) def testGetTime(self): @@ -80,7 +83,7 @@ class DateDetectorTest(LogCaptureTestCase): # is not correctly determined atm, since year is not present # in the log entry. Since this doesn't effect the operation # of fail2ban -- we just ignore incorrect day of the week - ( datelog, matchlog ) = self.__datedetector.getTime(log) + ( datelog, matchlog ) = self.datedetector.getTime(log) self.assertEqual(datelog, dateUnix) self.assertEqual(matchlog.group(), 'Jan 23 21:59:59') @@ -140,7 +143,7 @@ class DateDetectorTest(LogCaptureTestCase): if not bound and prefix == "word-boundary": continue logSys.debug(' -- test %-5s for %r', should_match, log) # with getTime: - logtime = self.__datedetector.getTime(log) + logtime = self.datedetector.getTime(log) if should_match: self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log)) @@ -152,8 +155,8 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) # with getTime(matchTime) - this combination used in filter: - (timeMatch, template) = matchTime = self.__datedetector.matchTime(log) - logtime = self.__datedetector.getTime(log, matchTime) + (timeMatch, template) = matchTime = self.datedetector.matchTime(log) + logtime = self.datedetector.getTime(log, matchTime) logSys.debug(' -- found - %r', template.name if timeMatch else False) if should_match: self.assertNotEqual(logtime, None, @@ -168,26 +171,26 @@ class DateDetectorTest(LogCaptureTestCase): logSys.debug(' -- OK') def testAllUniqueTemplateNames(self): - self.assertRaises(ValueError, self.__datedetector.appendTemplate, - self.__datedetector.templates[0]) + self.assertRaises(ValueError, self.datedetector.appendTemplate, + self.datedetector.templates[0]) def testFullYearMatch_gh130(self): # see https://github.com/fail2ban/fail2ban/pull/130 # yoh: unfortunately this test is not really effective to reproduce the # situation but left in place to assure consistent behavior mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).timetuple()) - logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') + logdate = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertNotEqual(logdate, None) ( logTime, logMatch ) = logdate self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') # confuse it with year being at the end for i in xrange(10): - ( logTime, logMatch ) = self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') + ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(), '11/10/2012 02:37:17') # and now back to the original - ( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') + ( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') @@ -199,8 +202,7 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(t.matchDate('aaaac').group(), 'aaaac') def testAmbiguousInOrderedTemplates(self): - dd = DateDetector() - dd.addDefaultTemplate() + dd = self.datedetector for (debit, line, cnt) in ( # shortest distance to datetime should win: ("030324 0:03:59", "some free text 030324 0:03:59 -- 2003-03-07 17:05:01 ...", 1), @@ -224,6 +226,25 @@ class DateDetectorTest(LogCaptureTestCase): self.assertTrue(match) self.assertEqual(match.group(), debit) + def testLowLevelLogging(self): + # test coverage for the deep (heavy) debug messages: + try: + self.__old_eff_level = datedetector.logLevel + if datedetector.logLevel < logSys.getEffectiveLevel()+1: + datedetector.logLevel = logSys.getEffectiveLevel()+1 + dd = self.datedetector + i = 0 + for (line, cnt) in ( + ("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 2), + ("server mysqld[5906]: 051007 06:10:%02i 5907 [Warning] Access denied", 5), + ("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 10), + ): + for i in range(i, i+cnt+1): + logSys.debug('== test: %r', (line % i, cnt)) + match, template = dd.matchTime(line % i) + self.assertTrue(match) + finally: + datedetector.logLevel = self.__old_eff_level iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") From ab0ac2111c4e123b7f685235be99f515938fd193 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 5 Oct 2016 19:34:21 +0200 Subject: [PATCH 11/91] added possibility to specify more precise default date pattern: - `datepattern = {^LN-BEG}` - only line-begin anchored default patterns (matches date only at begin of line, or with max distance up to 2 non-alphanumeric characters from line-begin); - `datepattern = {*WD-BEG}` - only word-begin anchored default patterns; - `datepattern = ^prefix{DATE}suffix` - exact specified default patterns (using prefix and suffix); common filter configs gets a more precise, line-begin anchored (datepattern = {^LN-BEG}) resp. custom anchoring default date-patterns; --- config/filter.d/3proxy.conf | 2 + config/filter.d/apache-common.conf | 2 + config/filter.d/apache-pass.conf | 4 -- config/filter.d/assp.conf | 2 + config/filter.d/asterisk.conf | 1 + config/filter.d/common.conf | 3 ++ config/filter.d/counter-strike.conf | 2 - config/filter.d/courier-auth.conf | 2 + config/filter.d/directadmin.conf | 1 - config/filter.d/freeswitch.conf | 2 + config/filter.d/kerio.conf | 2 - config/filter.d/monit.conf | 2 +- config/filter.d/murmur.conf | 3 +- config/filter.d/nginx-http-auth.conf | 2 + config/filter.d/nginx-limit-req.conf | 1 + config/filter.d/openhab.conf | 1 - config/filter.d/oracleims.conf | 4 +- config/filter.d/selinux-common.conf | 2 + config/filter.d/sogo-auth.conf | 4 +- config/filter.d/squirrelmail.conf | 2 - config/filter.d/sshd.conf | 4 +- fail2ban/client/fail2banregex.py | 5 +- fail2ban/client/filterreader.py | 17 ++++--- fail2ban/server/datedetector.py | 48 +++++++++++++------ fail2ban/server/datetemplate.py | 20 ++++++-- fail2ban/server/filter.py | 35 ++++++++++---- fail2ban/server/strptime.py | 1 + fail2ban/tests/files/filter.d/testcase01.conf | 1 - fail2ban/tests/servertestcase.py | 2 +- 29 files changed, 120 insertions(+), 57 deletions(-) diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf index 299c3a29..76c7573c 100644 --- a/config/filter.d/3proxy.conf +++ b/config/filter.d/3proxy.conf @@ -9,6 +9,8 @@ failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ :\d+ [\d.]+:\d+ \d+ \d+ \d+\s ignoreregex = +datepattern = {^LN-BEG} + # DEV Notes: # http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are # all authentication problems (%E field) diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index c843c92e..3d1f902b 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -10,6 +10,8 @@ after = apache-common.local _apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client (:\d{1,5})?\] +datepattern = {^LN-BEG} + # Common prefix for [error] apache messages which also would include # Depending on the version it could be # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf index dd00f953..7bdff9c0 100644 --- a/config/filter.d/apache-pass.conf +++ b/config/filter.d/apache-pass.conf @@ -3,10 +3,6 @@ # # The knocking request must have a referer. -[INCLUDES] - -before = apache-common.conf - [Definition] failregex = ^ - \w+ \[\] "GET HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$ diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 278e25cb..8d6367c9 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -20,6 +20,8 @@ failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\ ignoreregex = +datepattern = {^LN-BEG} + # DEV Notes: # V1 Examples matches: # Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 79bd1ff2..df55d288 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -31,6 +31,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo ignoreregex = +datepattern = {^LN-BEG} # Author: Xavier Devlamynck / Daniel Black # diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index 586f428a..a8cba188 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -61,4 +61,7 @@ __prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostn # pam_ldap __pam_auth = pam_unix +# standardly all formats using prefix have line-begin anchored date: +datepattern = {^LN-BEG} + # Author: Yaroslav Halchenko diff --git a/config/filter.d/counter-strike.conf b/config/filter.d/counter-strike.conf index a896b5ca..294927bc 100644 --- a/config/filter.d/counter-strike.conf +++ b/config/filter.d/counter-strike.conf @@ -8,8 +8,6 @@ failregex = ^: Bad Rcon: "rcon \d+ "\S+" sv_contact ".*?"" from ":\d+"$ ignoreregex = -[Init] - datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S diff --git a/config/filter.d/courier-auth.conf b/config/filter.d/courier-auth.conf index 1170a63a..87ee55b4 100644 --- a/config/filter.d/courier-auth.conf +++ b/config/filter.d/courier-auth.conf @@ -15,5 +15,7 @@ failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[\]$ ignoreregex = +datepattern = {^LN-BEG} + # Author: Christoph Haas # Modified by: Cyril Jaquier diff --git a/config/filter.d/directadmin.conf b/config/filter.d/directadmin.conf index 7622e548..87c7802f 100644 --- a/config/filter.d/directadmin.conf +++ b/config/filter.d/directadmin.conf @@ -13,7 +13,6 @@ failregex = ^: \'\' \d{1,3} failed login attempt(s)?. \s* ignoreregex = -[Init] datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S # diff --git a/config/filter.d/freeswitch.conf b/config/filter.d/freeswitch.conf index 29940240..4759fbed 100644 --- a/config/filter.d/freeswitch.conf +++ b/config/filter.d/freeswitch.conf @@ -26,6 +26,8 @@ failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challe ignoreregex = +datepattern = {^LN-BEG} + # Author: Rupa SChomaker, soapee01, Daniel Black # https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban # Thanks to Jim on mailing list of samples and guidance diff --git a/config/filter.d/kerio.conf b/config/filter.d/kerio.conf index 313c9b36..e0d94753 100644 --- a/config/filter.d/kerio.conf +++ b/config/filter.d/kerio.conf @@ -9,8 +9,6 @@ failregex = ^ SMTP Spam attack detected from , ignoreregex = -[Init] - datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\] # DEV NOTES: diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index 599bd363..b652a1f4 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -13,7 +13,7 @@ before = common.conf _daemon = monit # Regexp for previous (accessing monit httpd) and new (access denied) versions -failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ +failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ ^%(__prefix_line)s\w+: access denied -- client : (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$ # Ignore login with empty user (first connect, no user specified) diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf index 3775a9d2..507bbd2f 100644 --- a/config/filter.d/murmur.conf +++ b/config/filter.d/murmur.conf @@ -15,13 +15,14 @@ _daemon = murmurd # variable in your server config file (murmur.ini / mumble-server.ini). _usernameregex = [^>]+ -_prefix = [\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from :\d+: +_prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from :\d+: failregex = ^%(_prefix)s Invalid server password$ ^%(_prefix)s Wrong certificate or password for existing user$ ignoreregex = +datepattern = ^{DATE} # DEV Notes: # diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index a689f66a..e45a7f49 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -8,6 +8,8 @@ failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not f ignoreregex = +datepattern = {^LN-BEG} + # DEV NOTES: # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Extensive search of all nginx auth failures not done yet. diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index 13cf75c2..c283849c 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -43,3 +43,4 @@ failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by z ignoreregex = +datepattern = {^LN-BEG} diff --git a/config/filter.d/openhab.conf b/config/filter.d/openhab.conf index 83857c7a..f6b96339 100644 --- a/config/filter.d/openhab.conf +++ b/config/filter.d/openhab.conf @@ -9,7 +9,6 @@ [Definition] failregex = ^\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$ -[Init] datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z diff --git a/config/filter.d/oracleims.conf b/config/filter.d/oracleims.conf index e951ff5d..7d75c322 100644 --- a/config/filter.d/oracleims.conf +++ b/config/filter.d/oracleims.conf @@ -52,10 +52,12 @@ before = common.conf # Note that you MUST have LOG_FORMAT=4 for this to work! # -failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$ +failregex = tr="[A-Z]+\|[0-9.]+\|\d+\|\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. # Values: TEXT # ignoreregex = + +datepattern = ^' for user '.*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$ -ignoreregex = +ignoreregex = "^" + +datepattern = {^LN-BEG} # # DEV Notes: diff --git a/config/filter.d/squirrelmail.conf b/config/filter.d/squirrelmail.conf index af0c38e7..31e922e6 100644 --- a/config/filter.d/squirrelmail.conf +++ b/config/filter.d/squirrelmail.conf @@ -5,8 +5,6 @@ failregex = ^ \[LOGIN_ERROR\].*from : Unknown user or password incorrect\. ignoreregex = -[Init] - datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S # DEV NOTES: diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index eeb1518e..c1c8eacf 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -38,13 +38,13 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ignoreregex = -[Init] - # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 10 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd +datepattern = {^LN-BEG} + # DEV Notes: # # "Failed \S+ for .*? from ..." failregex uses non-greedy catch-all because diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 6680b097..bf49e7a6 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -474,8 +474,9 @@ class Fail2banRegex(object): if self._verbose or template.hits: out.append("[%d] %s" % (template.hits, template.name)) if self._verbose_date: - out.append(" # weight: %3s, pattern: %s" % ( - template.weight, getattr(template, 'pattern', ''),)) + out.append(" # weight: %.3f (%.3f), pattern: %s" % ( + template.weight, template.template.weight, + getattr(template, 'pattern', ''),)) out.append(" # regex: %s" % (getattr(template, 'regex', ''),)) pprint_list(out, "[# of hits] date format") diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b30f914..9bd94fd6 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -40,6 +40,9 @@ class FilterReader(DefinitionInitConfigReader): _configOpts = { "ignoreregex": ["string", None], "failregex": ["string", ""], + "maxlines": ["int", None], + "datepattern": ["string", None], + "journalmatch": ["string", None], } def setFile(self, fileName): @@ -74,16 +77,16 @@ class FilterReader(DefinitionInitConfigReader): stream.append(["multi-set", self._jailName, "add" + opt, multi]) elif len(multi): stream.append(["set", self._jailName, "add" + opt, multi[0]]) - if self._initOpts: - if 'maxlines' in self._initOpts: + elif opt == 'maxlines': # We warn when multiline regex is used without maxlines > 1 # therefore keep sure we set this option first. - stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) - if 'datepattern' in self._initOpts: - stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]]) + stream.insert(0, ["set", self._jailName, "maxlines", value]) + elif opt == 'datepattern': + stream.append(["set", self._jailName, "datepattern", value]) # Do not send a command if the match is empty. - if self._initOpts.get("journalmatch", '') != '': - for match in self._initOpts["journalmatch"].split("\n"): + elif opt == 'journalmatch': + for match in value.split("\n"): + if match == '': continue stream.append( ["set", self._jailName, "addjournalmatch"] + shlex.split(match)) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index ba2b6377..0aac4337 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -26,7 +26,7 @@ import time from threading import Lock -from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch +from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch from ..helpers import getLogger # Gets the instance of the logger. @@ -34,6 +34,8 @@ logSys = getLogger(__name__) logLevel = 6 +RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE) + class DateDetectorCache(object): """Implements the caching of the default templates list. @@ -54,15 +56,18 @@ class DateDetectorCache(object): self._addDefaultTemplate() return self.__templates - def _cacheTemplate(self, template): + def _cacheTemplate(self, template, lineBeginOnly=False): """Cache Fail2Ban's default template. """ if isinstance(template, str): # exact given template with word benin-end boundary: - template = DatePatternRegex(template) + if not lineBeginOnly: + template = DatePatternRegex(template) + else: + template = DatePatternRegex(template, wordBegin='start') # additional template, that prefers datetime at start of a line (safety+performance feature): - if hasattr(template, 'regex'): + if not lineBeginOnly and hasattr(template, 'regex'): template2 = copy.copy(template) regex = getattr(template, 'pattern', template.regex) template2.setRegex(regex, wordBegin='start', wordEnd=True) @@ -90,6 +95,7 @@ class DateDetectorCache(object): # prefixed with optional time zone (monit): # PDT Apr 16 21:05:29 self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 @@ -114,17 +120,18 @@ class DateDetectorCache(object): # TAI64N self._cacheTemplate(DateTai64n()) # Epoch + self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True) self._cacheTemplate(DateEpoch()) # Only time information in the log - self._cacheTemplate("^%H:%M:%S") + self._cacheTemplate("%H:%M:%S", lineBeginOnly=True) # <09/16/08@05:03:30> - self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>") + self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True) # MySQL: 130322 11:46:11 self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S") # Apache Tomcat self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 - self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") + self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True) self.__templates = self.__tmpcache[0] + self.__tmpcache[1] del self.__tmpcache @@ -172,6 +179,8 @@ class DateDetector(object): self.__lastTemplIdx = 0x7fffffff # first free place: self.__firstUnused = 0 + # pre-match pattern: + self.__preMatch = None def _appendTemplate(self, template): name = template.name @@ -200,10 +209,18 @@ class DateDetector(object): template = DatePatternRegex(template) self._appendTemplate(template) - def addDefaultTemplate(self): + def addDefaultTemplate(self, filterTemplate=None, preMatch=None): """Add Fail2Ban's default set of date templates. """ for template in DateDetector._defCache.templates: + # filter if specified: + if filterTemplate is not None and not filterTemplate(template): continue + # if exact pattern available - create copy of template, contains replaced {DATE} with default regex: + if preMatch is not None: + regex = getattr(template, 'pattern', template.regex) + template = copy.copy(template) + template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) + # append date detector template: self._appendTemplate(template) @property @@ -230,7 +247,7 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ - #logSys.log(logLevel, "try to match time for line: %.250s", line) + #logSys.log(logLevel, "try to match time for line: %.120s", line) match = None # first try to use last template with same start/end position: i = self.__lastTemplIdx @@ -238,13 +255,16 @@ class DateDetector(object): ddtempl = self.__templates[i] template = ddtempl.template distance, endpos = self.__lastPos[0], self.__lastEndPos[0] + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", + i, distance, endpos, + line[distance-1:distance], self.__lastPos[1], + line[distance:endpos], + line[endpos:endpos+1], self.__lastEndPos[1]) # check same boundaries left/right, otherwise possible collision/pattern switch: if (line[distance-1:distance] == self.__lastPos[1] and line[endpos:endpos+1] == self.__lastEndPos[1] ): - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ... %s ...", - i, distance, endpos, line[distance:endpos]) match = template.matchDate(line, distance, endpos) if match: distance = match.start() @@ -270,8 +290,8 @@ class DateDetector(object): if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", i, distance, ddtempl.distance, self.__lastPos[0], template.name) - ## if line-begin anchored - stop searching: - if template.flags & DateTemplate.LINE_BEGIN: + ## if line-begin/end anchored - stop searching: + if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): break ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index d85c8f66..fcd51f99 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -33,9 +33,12 @@ from ..helpers import getLogger logSys = getLogger(__name__) RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)') -RE_NO_WRD_BOUND_END = re.compile(r'(?(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))", - wordBegin=False) ;# already line begin resp. word begin anchored + if not lineBeginOnly: + regex = r"(?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" + self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored + else: + regex = r"(?P(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))" + self.setRegex(regex, wordBegin='start', wordEnd=True) def getDate(self, line, dateMatch=None): """Method to return the date for a log line. diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9b1aefd6..cbf59fcb 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -35,7 +35,7 @@ from .ipdns import DNSUtils, IPAddr from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n +from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -257,14 +257,29 @@ class Filter(JailThread): if pattern is None: self.dateDetector = None return - elif pattern.upper() == "EPOCH": - template = DateEpoch() - template.name = "Epoch" - elif pattern.upper() == "TAI64N": - template = DateTai64n() - template.name = "TAI64N" else: - template = DatePatternRegex(pattern) + key = pattern.upper() + if key == "EPOCH": + template = DateEpoch() + template.name = "Epoch" + elif key == "TAI64N": + template = DateTai64n() + template.name = "TAI64N" + elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): + self.dateDetector = DateDetector() + flt = \ + lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ + lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ + None + self.dateDetector.addDefaultTemplate(flt) + return + elif "{DATE}" in key: + self.dateDetector = DateDetector() + self.dateDetector.addDefaultTemplate( + lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) + return + else: + template = DatePatternRegex(pattern) self.dateDetector = DateDetector() self.dateDetector.appendTemplate(template) logSys.info(" date pattern `%r`: `%s`", @@ -280,9 +295,9 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: templates = self.dateDetector.templates - if len(templates) > 1: + if len(templates) > 2: return None, "Default Detectors" - elif len(templates) == 1: + elif len(templates): if hasattr(templates[0], "pattern"): pattern = templates[0].pattern else: diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 85045ed8..da00a57e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -40,6 +40,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): #timeRE['z'] = r"%s?(?PZ|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] +timeRE['Z'] = r"(?P[A-Z]{3,5})" timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" # Extend build-in TimeRE with some exact patterns diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf index 0dbbbd2a..b3700d0b 100644 --- a/fail2ban/tests/files/filter.d/testcase01.conf +++ b/fail2ban/tests/files/filter.d/testcase01.conf @@ -33,7 +33,6 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro # ignoreregex = ^.+ john from host 192.168.1.1\s*$ -[Init] # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 1 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index d4320257..cd7ce5c9 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1107,7 +1107,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): # (we don't use it in this test at all): elif unittest.F2B.fast and ( len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex' - ): + ): # pragma: no cover cmd[0] = "set" cmd[3] = "DUMMY-REGEX " # command to server, use cmdHandler direct instead of `transm.proceed(cmd)`: From e735f8f56835abeda6135c2088b54753193de7c9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 6 Oct 2016 15:27:49 +0200 Subject: [PATCH 12/91] default non-unicode and case-sensitive matching (by pattern templates automatically add `(?iu)` for "ignore case" and "unicode" if expected) --- fail2ban/server/datedetector.py | 47 ++++++++++++++------------ fail2ban/server/datetemplate.py | 25 ++++++++++---- fail2ban/tests/datedetectortestcase.py | 45 +++++++++++++++++++++--- 3 files changed, 85 insertions(+), 32 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 0aac4337..f256a425 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -254,27 +254,32 @@ class DateDetector(object): if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template - distance, endpos = self.__lastPos[0], self.__lastEndPos[0] - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", - i, distance, endpos, - line[distance-1:distance], self.__lastPos[1], - line[distance:endpos], - line[endpos:endpos+1], self.__lastEndPos[1]) - # check same boundaries left/right, otherwise possible collision/pattern switch: - if (line[distance-1:distance] == self.__lastPos[1] and - line[endpos:endpos+1] == self.__lastEndPos[1] - ): - match = template.matchDate(line, distance, endpos) - if match: - distance = match.start() - endpos = match.end() - # if different position, possible collision/pattern switch: - if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]: - logSys.log(logLevel, " matched last time template #%02i", i) - else: - logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") - match = None + if template.flags & DateTemplate.LINE_BEGIN: + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) + match = template.matchDate(line) + else: + distance, endpos = self.__lastPos[0], self.__lastEndPos[0] + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", + i, distance, endpos, + line[distance-1:distance], self.__lastPos[1], + line[distance:endpos], + line[endpos:endpos+1], self.__lastEndPos[1]) + # check same boundaries left/right, otherwise possible collision/pattern switch: + if (line[distance-1:distance] == self.__lastPos[1] and + line[endpos:endpos+1] == self.__lastEndPos[1] + ): + match = template.matchDate(line, distance, endpos) + if match: + distance = match.start() + endpos = match.end() + # if different position, possible collision/pattern switch: + if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]: + logSys.log(logLevel, " matched last time template #%02i", i) + else: + logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") + match = None # search template and better match: if not match: self.__lastTemplIdx = 0x7fffffff diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index fcd51f99..9524e373 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -32,13 +32,18 @@ from ..helpers import getLogger logSys = getLogger(__name__) -RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)') +RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)') RE_NO_WRD_BOUND_END = re.compile(r'(? Date: Thu, 6 Oct 2016 15:39:38 +0200 Subject: [PATCH 13/91] speedup SeekToTime test cases using exact date pattern... --- fail2ban/tests/filtertestcase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 6c1a637c..ea996072 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -476,6 +476,8 @@ class LogFileFilterPoll(unittest.TestCase): self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME)) def testSeekToTimeSmallFile(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') @@ -560,6 +562,8 @@ class LogFileFilterPoll(unittest.TestCase): _killfile(f, fname) def testSeekToTimeLargeFile(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') From bd1eb70c52f078fea65d1688915487a5dea1f3c7 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 6 Oct 2016 16:44:07 +0200 Subject: [PATCH 14/91] speedup template first time selection through pre-sorted template list by template hits --- fail2ban/server/datedetector.py | 15 +++++++++------ fail2ban/server/datetemplate.py | 3 +++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index f256a425..7cbc0f5c 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -88,14 +88,9 @@ class DateDetectorCache(object): # simple date: 2005/01/23 21:59:59 # custom for syslog-ng 2006.12.21 06:43:20 self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") - # 20050123T215959, 20050123 215959 - self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - # prefixed with optional time zone (monit): - # PDT Apr 16 21:05:29 self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") - self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 @@ -132,6 +127,12 @@ class DateDetectorCache(object): self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True) + # 20050123T215959, 20050123 215959 + self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") + # prefixed with optional named time zone (monit): + # PDT Apr 16 21:05:29 + self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + # self.__templates = self.__tmpcache[0] + self.__tmpcache[1] del self.__tmpcache @@ -212,7 +213,9 @@ class DateDetector(object): def addDefaultTemplate(self, filterTemplate=None, preMatch=None): """Add Fail2Ban's default set of date templates. """ - for template in DateDetector._defCache.templates: + for template in sorted(DateDetector._defCache.templates, + lambda a,b: b.hits - a.hits + ): # filter if specified: if filterTemplate is not None and not filterTemplate(template): continue # if exact pattern available - create copy of template, contains replaced {DATE} with default regex: diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 9524e373..70a4b1b5 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -65,6 +65,7 @@ class DateTemplate(object): self.name = "" self.weight = 1.0 self.flags = 0 + self.hits = 0 self._regex = "" self._cRegex = None @@ -133,6 +134,8 @@ class DateTemplate(object): if not self._cRegex: self._compileRegex() dateMatch = self._cRegex.search(line, *args); # pos, endpos + if dateMatch: + self.hits += 1 return dateMatch @abstractmethod From ae7297e16b0188953bcdd18967bd22982b7fbfbb Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 Oct 2016 14:57:45 +0200 Subject: [PATCH 15/91] more precise date template handling (WARNING: this commit creates possible incompatibilities): - datedetector rewritten more strict as earlier; - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`); - more as one date pattern can be specified using option `datepattern` now (new-line separated); - some default options like `datepattern` can be specified directly in section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]` section, because of performance (each extra section costs time); - option `datepattern` can be specified in jail also (jails without filters); - if first group specified, only this will be cut out from search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match pattern, and leaves `date:[] failure ip...` for searching in filter); - faster match and fewer searching of appropriate templates (DateDetector.matchTime calls rarer DateTemplate.matchDate now); - standard filters extended with exact prefixed or anchored date templates; template cache introduced (in opposition to default template cache, holds custom templates cached by pattern for possible common usage of same template/regex); --- config/filter.d/apache-badbots.conf | 3 + config/filter.d/apache-fakegooglebot.conf | 2 + config/filter.d/apache-pass.conf | 3 + config/filter.d/assp.conf | 3 +- config/filter.d/dovecot.conf | 5 +- config/filter.d/ejabberd-auth.conf | 4 +- config/filter.d/guacamole.conf | 5 +- config/filter.d/nginx-botsearch.conf | 3 + config/filter.d/nsd.conf | 3 + config/filter.d/php-url-fopen.conf | 3 + config/filter.d/portsentry.conf | 3 + config/filter.d/sogo-auth.conf | 5 +- config/filter.d/squid.conf | 3 + config/filter.d/tine20.conf | 3 + fail2ban/client/jailreader.py | 17 +--- fail2ban/server/datedetector.py | 99 ++++++++++++++----- fail2ban/server/datetemplate.py | 59 +++++++---- fail2ban/server/filter.py | 46 +++------ fail2ban/server/strptime.py | 3 - fail2ban/server/transmitter.py | 2 +- fail2ban/tests/action_d/test_badips.py | 1 + fail2ban/tests/action_d/test_smtp.py | 1 + fail2ban/tests/banmanagertestcase.py | 2 + fail2ban/tests/clientbeautifiertestcase.py | 1 + fail2ban/tests/clientreadertestcase.py | 1 + .../config/filter.d/zzz-generic-example.conf | 5 + fail2ban/tests/datedetectortestcase.py | 59 +++++------ fail2ban/tests/fail2banclienttestcase.py | 1 + fail2ban/tests/fail2banregextestcase.py | 9 ++ fail2ban/tests/failmanagertestcase.py | 1 + fail2ban/tests/files/logs/zzz-generic-example | 8 +- fail2ban/tests/filtertestcase.py | 24 +++++ fail2ban/tests/misctestcase.py | 1 + fail2ban/tests/samplestestcase.py | 1 + fail2ban/tests/servertestcase.py | 8 +- fail2ban/tests/sockettestcase.py | 1 + fail2ban/tests/utils.py | 18 +++- 37 files changed, 280 insertions(+), 136 deletions(-) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index 48b30666..f42aa159 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -14,6 +14,9 @@ failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} + # DEV Notes: # List of bad bots fetched from http://www.user-agents.org # Generated on Thu Nov 7 14:23:35 PST 2013 by files/gen_badbots. diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf index b8a73504..729410ad 100644 --- a/config/filter.d/apache-fakegooglebot.conf +++ b/config/filter.d/apache-fakegooglebot.conf @@ -6,6 +6,8 @@ failregex = ^ .*Googlebot.*$ ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} # DEV Notes: # diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf index 7bdff9c0..3cab87b0 100644 --- a/config/filter.d/apache-pass.conf +++ b/config/filter.d/apache-pass.conf @@ -9,6 +9,9 @@ failregex = ^ - \w+ \[\] "GET HTTP/1\.[01]" 200 \d+ ".*" "[ ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} + [Init] knocking_url = /knocking/ diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 8d6367c9..7cca81d2 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -20,7 +20,8 @@ failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\ ignoreregex = -datepattern = {^LN-BEG} +datepattern = {^LN-BEG}%%b-%%d-%%Exy %%H:%%M:%%S + {^LN-BEG} # DEV Notes: # V1 Examples matches: diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 136a3947..6f8510fc 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -17,10 +17,11 @@ failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentica ignoreregex = -[Init] - journalmatch = _SYSTEMD_UNIT=dovecot.service +datepattern = {^LN-BEG}TAI64N + {^LN-BEG} + # DEV Notes: # * the first regex is essentially a copy of pam-generic.conf # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016) diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf index 512fdb7c..edd87463 100644 --- a/config/filter.d/ejabberd-auth.conf +++ b/config/filter.d/ejabberd-auth.conf @@ -25,8 +25,6 @@ failregex = ^=INFO REPORT==== ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\ # ignoreregex = -[Init] - # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 @@ -35,3 +33,5 @@ maxlines = 2 # Values: TEXT # journalmatch = + +datepattern = ^(?:=[^=]+={3,} )?({DATE}) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 49cecc5a..09b4e7b0 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -17,6 +17,9 @@ failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" fa # ignoreregex = -[Init] # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 + +datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p + ^WARNING:()** + {^LN-BEG} \ No newline at end of file diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 6853e1e8..0be895b2 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -13,6 +13,9 @@ failregex = ^ \- \S+ \[\] \"(GET|POST|HEAD) \/ \S+\" 404 .+$ ignoreregex = +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + ^[^\[]*\[({DATE}) + {^LN-BEG} # DEV Notes: # Based on apache-botsearch filter diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index 8f32f7be..bfd99544 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -26,3 +26,6 @@ failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ ^%(__prefix_line)sinfo: .* refused, no acl matches\.$ ignoreregex = + +datepattern = {^LN-BEG}Epoch + {^LN-BEG} \ No newline at end of file diff --git a/config/filter.d/php-url-fopen.conf b/config/filter.d/php-url-fopen.conf index 87bd04c8..a7957c9d 100644 --- a/config/filter.d/php-url-fopen.conf +++ b/config/filter.d/php-url-fopen.conf @@ -18,3 +18,6 @@ ignoreregex = # http://blogs.buanzo.com.ar/2009/04/fail2ban-filter-for-php-injection-attacks.html#comment-1489 # # Author: Arturo 'Buanzo' Busleiman + +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} diff --git a/config/filter.d/portsentry.conf b/config/filter.d/portsentry.conf index 27dca9b4..35ca2a3d 100644 --- a/config/filter.d/portsentry.conf +++ b/config/filter.d/portsentry.conf @@ -8,5 +8,8 @@ failregex = \/ Port\: [0-9]+ (TCP|UDP) Blocked$ ignoreregex = +datepattern = {^LN-BEG}Epoch + {^LN-BEG} + # Author: Pacop diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf index 1a3d5292..48221dc0 100644 --- a/config/filter.d/sogo-auth.conf +++ b/config/filter.d/sogo-auth.conf @@ -8,7 +8,10 @@ failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '' for user '.*' migh ignoreregex = "^" -datepattern = {^LN-BEG} +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)? + ^[^\[]*\[({DATE}) + {^LN-BEG} # # DEV Notes: diff --git a/config/filter.d/squid.conf b/config/filter.d/squid.conf index e26cab9c..58694c48 100644 --- a/config/filter.d/squid.conf +++ b/config/filter.d/squid.conf @@ -9,5 +9,8 @@ failregex = ^\s+\d\s\s+[A-Z_]+_DENIED/403 .*$ ignoreregex = +datepattern = {^LN-BEG}Epoch + {^LN-BEG} + # Author: Daniel Black diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf index 0fa6eccd..a80d89e8 100644 --- a/config/filter.d/tine20.conf +++ b/config/filter.d/tine20.conf @@ -10,6 +10,9 @@ failregex = ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0 ignoreregex = +datepattern = ^[^-]+ -- [^-]+ -- - ({DATE}) + {^LN-BEG} + # Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766 # Editor: Daniel Black # Advisor: Lars Kneschke diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9d01a693..3849c027 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -112,6 +112,7 @@ class JailReader(ConfigReader): ["string", "ignorecommand", None], ["string", "ignoreip", None], ["string", "filter", ""], + ["string", "datepattern", None], ["string", "action", ""]] # Before interpolation (substitution) add static options always available as default: @@ -195,6 +196,8 @@ class JailReader(ConfigReader): """ stream = [] + if self.__filter: + stream.extend(self.__filter.convert()) for opt, value in self.__opts.iteritems(): if opt == "logpath" and \ not self.__opts.get('backend', None).startswith("systemd"): @@ -216,17 +219,9 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "logencoding", value]) elif opt == "backend": backend = value - elif opt == "maxretry": - stream.append(["set", self.__name, "maxretry", value]) elif opt == "ignoreip": for ip in splitwords(value): stream.append(["set", self.__name, "addignoreip", ip]) - elif opt == "findtime": - stream.append(["set", self.__name, "findtime", value]) - elif opt == "bantime": - stream.append(["set", self.__name, "bantime", value]) - elif opt == "usedns": - stream.append(["set", self.__name, "usedns", value]) elif opt in ("failregex", "ignoreregex"): multi = [] for regex in value.split('\n'): @@ -237,10 +232,8 @@ class JailReader(ConfigReader): stream.append(["multi-set", self.__name, "add" + opt, multi]) elif len(multi): stream.append(["set", self.__name, "add" + opt, multi[0]]) - elif opt == "ignorecommand": - stream.append(["set", self.__name, "ignorecommand", value]) - if self.__filter: - stream.extend(self.__filter.convert()) + elif opt not in ('action', 'filter', 'enabled'): + stream.append(["set", self.__name, opt, value]) for action in self.__actions: if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 7cbc0f5c..cced6d91 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -27,6 +27,7 @@ import time from threading import Lock from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch +from .utils import Utils from ..helpers import getLogger # Gets the instance of the logger. @@ -90,7 +91,7 @@ class DateDetectorCache(object): self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 @@ -112,8 +113,6 @@ class DateDetectorCache(object): # subseconds explicit to avoid possible %m<->%d confusion # with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?") - # TAI64N - self._cacheTemplate(DateTai64n()) # Epoch self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True) self._cacheTemplate(DateEpoch()) @@ -132,6 +131,10 @@ class DateDetectorCache(object): # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + # +00:00 Jan 23 21:59:59.011 2005 + self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + # TAI64N + self._cacheTemplate(DateTai64n()) # self.__templates = self.__tmpcache[0] + self.__tmpcache[1] del self.__tmpcache @@ -168,6 +171,7 @@ class DateDetector(object): templates """ _defCache = DateDetectorCache() + _patternCache = Utils.Cache(maxCount=1000, maxTime=60*60) def __init__(self): self.__templates = list() @@ -183,9 +187,10 @@ class DateDetector(object): # pre-match pattern: self.__preMatch = None - def _appendTemplate(self, template): + def _appendTemplate(self, template, ignoreDup=False): name = template.name if name in self.__known_names: + if ignoreDup: return raise ValueError( "There is already a template with name %s" % name) self.__known_names.add(name) @@ -207,24 +212,56 @@ class DateDetector(object): If a template already exists with the same name. """ if isinstance(template, str): - template = DatePatternRegex(template) + key = pattern = template + if '%' not in pattern: + key = pattern.upper() + template = DateDetector._patternCache.get(key) + + if not template: + if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"): + template = DateEpoch(lineBeginOnly=(key != "EPOCH")) + elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"): + template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False)) + elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): + flt = \ + lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ + lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ + None + self.addDefaultTemplate(flt) + return + elif "{DATE}" in key: + self.addDefaultTemplate( + lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) + return + else: + template = DatePatternRegex(pattern) + + DateDetector._patternCache.set(key, template) + self._appendTemplate(template) + logSys.info(" date pattern `%r`: `%s`", + getattr(template, 'pattern', ''), template.name) + logSys.debug(" date pattern regex for %r: %s", + getattr(template, 'pattern', ''), template.regex) def addDefaultTemplate(self, filterTemplate=None, preMatch=None): """Add Fail2Ban's default set of date templates. """ - for template in sorted(DateDetector._defCache.templates, - lambda a,b: b.hits - a.hits - ): + ignoreDup = len(self.__templates) > 0 + for template in DateDetector._defCache.templates: # filter if specified: if filterTemplate is not None and not filterTemplate(template): continue # if exact pattern available - create copy of template, contains replaced {DATE} with default regex: if preMatch is not None: - regex = getattr(template, 'pattern', template.regex) - template = copy.copy(template) - template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) - # append date detector template: - self._appendTemplate(template) + deftemplate = template + template = DateDetector._patternCache.get((preMatch, deftemplate.name)) + if not template: + regex = getattr(deftemplate, 'pattern', deftemplate.regex) + template = copy.copy(deftemplate) + template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) + DateDetector._patternCache.set((preMatch, deftemplate.name), template) + # append date detector template (ignore duplicate if some was added before default): + self._appendTemplate(template, ignoreDup=ignoreDup) @property def templates(self): @@ -250,17 +287,22 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ - #logSys.log(logLevel, "try to match time for line: %.120s", line) + # if no templates specified - default templates should be used: + if not len(self.__templates): + self.addDefaultTemplate() + logSys.log(logLevel-1, "try to match time for line: %.120s", line) match = None # first try to use last template with same start/end position: + ignoreBySearch = 0x7fffffff i = self.__lastTemplIdx if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template - if template.flags & DateTemplate.LINE_BEGIN: + if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): if logSys.getEffectiveLevel() <= logLevel-1: logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) + ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] if logSys.getEffectiveLevel() <= logLevel-1: @@ -278,18 +320,28 @@ class DateDetector(object): distance = match.start() endpos = match.end() # if different position, possible collision/pattern switch: - if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]: + if ( + template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or + (distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]) + ): logSys.log(logLevel, " matched last time template #%02i", i) else: logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") match = None + else: + logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: self.__lastTemplIdx = 0x7fffffff - logSys.log(logLevel, " search template ...") + logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) found = None, 0x7fffffff, -1 i = 0 for ddtempl in self.__templates: + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) + if i == ignoreBySearch: + i += 1 + continue template = ddtempl.template match = template.matchDate(line) if match: @@ -298,15 +350,18 @@ class DateDetector(object): if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", i, distance, ddtempl.distance, self.__lastPos[0], template.name) + ## last (or single) template - fast stop: + if i+1 >= len(self.__templates): + break ## if line-begin/end anchored - stop searching: if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): break + ## stop searching if next template still unused, but we had already hits: + if (distance == 0 and ddtempl.hits) and not self.__templates[i+1].template.hits: + break ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: - if ( - (distance > ddtempl.distance or distance > self.__lastPos[0]) and - len(self.__templates) > 1 - ): + if distance > ddtempl.distance or distance > self.__lastPos[0]: logSys.log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: @@ -374,7 +429,7 @@ class DateDetector(object): if date is not None: if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " got time %f for %r using template %s", - date[0], date[1].group(), template.name) + date[0], date[1].group(1), template.name) return date except ValueError: pass diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 70a4b1b5..3a9612aa 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import re +import re, time from abc import abstractmethod from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE @@ -32,9 +32,14 @@ from ..helpers import getLogger logSys = getLogger(__name__) -RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)') -RE_NO_WRD_BOUND_END = re.compile(r'(?(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" + regex = r"((?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored else: - regex = r"(?P(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))" + regex = r"((?P(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))" self.setRegex(regex, wordBegin='start', wordEnd=True) def getDate(self, line, dateMatch=None): @@ -199,7 +219,7 @@ class DateEpoch(DateTemplate): dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch - return (float(dateMatch.group()), dateMatch) + return (float(dateMatch.group(1)), dateMatch) return None @@ -244,7 +264,13 @@ class DatePatternRegex(DateTemplate): self.setRegex(pattern) def setRegex(self, pattern, wordBegin=True, wordEnd=True): + # original pattern: self._pattern = pattern + # if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin: + if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern): + pattern = RE_EXLINE_BOUND_BEG.sub('', pattern) + wordBegin = 'start' + # wrap to regex: fmt = self._patternRE.sub(r'%(\1)s', pattern) self.name = fmt % self._patternName regex = fmt % timeRE @@ -285,12 +311,11 @@ class DateTai64n(DateTemplate): regex """ - def __init__(self): + def __init__(self, wordBegin=False): DateTemplate.__init__(self) self.name = "TAI64N" # We already know the format for TAI64N - # yoh: we should not add an additional front anchor - self.setRegex("@[0-9a-f]{24}", wordBegin=False) + self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin) def getDate(self, line, dateMatch=None): """Method to return the date for a log line. @@ -310,7 +335,7 @@ class DateTai64n(DateTemplate): dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch - value = dateMatch.group() + value = dateMatch.group(1) seconds_since_epoch = value[2:17] # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cbf59fcb..7f7584cf 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -35,7 +35,6 @@ from .ipdns import DNSUtils, IPAddr from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -94,7 +93,6 @@ class Filter(JailThread): self.ticks = 0 self.dateDetector = DateDetector() - self.dateDetector.addDefaultTemplate() logSys.debug("Created %s" % self) def __repr__(self): @@ -258,34 +256,12 @@ class Filter(JailThread): self.dateDetector = None return else: - key = pattern.upper() - if key == "EPOCH": - template = DateEpoch() - template.name = "Epoch" - elif key == "TAI64N": - template = DateTai64n() - template.name = "TAI64N" - elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): - self.dateDetector = DateDetector() - flt = \ - lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ - lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ - None - self.dateDetector.addDefaultTemplate(flt) - return - elif "{DATE}" in key: - self.dateDetector = DateDetector() - self.dateDetector.addDefaultTemplate( - lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) - return - else: - template = DatePatternRegex(pattern) - self.dateDetector = DateDetector() - self.dateDetector.appendTemplate(template) - logSys.info(" date pattern `%r`: `%s`", - pattern, template.name) - logSys.debug(" date pattern regex for %r: %s", - pattern, template.regex) + dd = DateDetector() + if not isinstance(pattern, (list, tuple)): + pattern = filter(bool, map(str.strip, re.split('\n+', pattern))) + for pattern in pattern: + dd.appendTemplate(pattern) + self.dateDetector = dd ## # Get the date detector pattern, or Default Detectors if not changed @@ -295,7 +271,8 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: templates = self.dateDetector.templates - if len(templates) > 2: + # lazy template init, by first match + if not len(templates) or len(templates) > 2: return None, "Default Detectors" elif len(templates): if hasattr(templates[0], "pattern"): @@ -303,6 +280,7 @@ class Filter(JailThread): else: pattern = None return pattern, templates[0].name + return None ## # Set the maximum retry value. @@ -483,9 +461,9 @@ class Filter(JailThread): (timeMatch, template) = self.dateDetector.matchTime(l) if timeMatch: tupleLine = ( - l[:timeMatch.start()], - l[timeMatch.start():timeMatch.end()], - l[timeMatch.end():], + l[:timeMatch.start(1)], + l[timeMatch.start(1):timeMatch.end(1)], + l[timeMatch.end(1):], (timeMatch, template) ) else: diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index da00a57e..643984d3 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -55,8 +55,6 @@ timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" # respect possible run in the test-cases (alternate date used there): timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) -# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates: -timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))" def getTimePatternRE(): keys = timeRE.keys() @@ -70,7 +68,6 @@ def getTimePatternRE(): 'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek", 'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%", 'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name", - 'ExLB': '{^LN-BEG}', } for key in set(keys) - set(names): # may not have them all... if key.startswith('Ex'): diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2f5be043..a0dfc639 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -303,7 +303,7 @@ class Transmitter: actionvalue = command[4] setattr(action, actionkey, actionvalue) return getattr(action, actionkey) - raise Exception("Invalid command (no set action or not yet implemented)") + raise Exception("Invalid command %r (no set action or not yet implemented)" % (command[1],)) def __commandGet(self, command): name = command[0] diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index 64db44e7..2f3b6723 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -32,6 +32,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable def setUp(self): """Call before every test case.""" + super(BadIPsActionTest, self).setUp() unittest.F2B.SkipIfNoNetwork() self.jail = DummyJail() diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 5c8b1923..dc28d5c0 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -45,6 +45,7 @@ class SMTPActionTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(SMTPActionTest, self).setUp() self.jail = DummyJail() pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py") pythonModuleName = os.path.basename(pythonModule.rstrip(".py")) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 4d964425..2c9d6601 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -32,6 +32,7 @@ from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(AddFailure, self).setUp() self.__ticket = BanTicket('193.168.0.128', 1167605999.0) self.__banManager = BanManager() @@ -134,6 +135,7 @@ class AddFailure(unittest.TestCase): class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() self.__ban_ip = "93.184.216.34" self.__asn = "15133" diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py index 0390a8ff..7d20e84e 100644 --- a/fail2ban/tests/clientbeautifiertestcase.py +++ b/fail2ban/tests/clientbeautifiertestcase.py @@ -32,6 +32,7 @@ class BeautifierTest(unittest.TestCase): def setUp(self): """ Call before every test case """ + super(BeautifierTest, self).setUp() self.b = Beautifier() def tearDown(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e68523c2..36722ff1 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -55,6 +55,7 @@ class ConfigReaderTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(ConfigReaderTest, self).setUp() self.d = tempfile.mkdtemp(prefix="f2b-temp") self.c = ConfigReaderUnshared(basedir=self.d) diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf index df30d725..8d1f2ae9 100644 --- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf +++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf @@ -20,3 +20,8 @@ failregex = ^%(__prefix_line)sF2B: failure from $ # just to test multiple ignoreregex: ignoreregex = ^%(__prefix_line)sF2B: error from 192.0.2.251$ ^%(__prefix_line)sF2B: error from 192.0.2.252$ + +# specify only exact date patterns, +1 with %%Y to test usage of last known date by wrong dates like 0000-00-00... +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)? + {^LN-BEG}%%Y(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 6bf92582..fdb7ec71 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -58,6 +58,8 @@ class DateDetectorTest(LogCaptureTestCase): return self.__datedetector def testGetEpochTime(self): + self.__datedetector = DateDetector() + self.__datedetector.appendTemplate('EPOCH') # correct epoch time, using all variants: for dateUnix in (1138049999, 32535244799): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): @@ -67,7 +69,7 @@ class DateDetectorTest(LogCaptureTestCase): self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) - self.assertIn(matchlog.group(), (str(dateUnix), str(dateUnix)+'.555')) + self.assertIn(matchlog.group(1), (str(dateUnix), str(dateUnix)+'.555')) # wrong, no epoch time (< 10 digits, more as 11 digits, begin/end of word) : for dateUnix in ('123456789', '9999999999999999', '1138049999A', 'A1138049999'): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): @@ -85,7 +87,7 @@ class DateDetectorTest(LogCaptureTestCase): # of fail2ban -- we just ignore incorrect day of the week ( datelog, matchlog ) = self.datedetector.getTime(log) self.assertEqual(datelog, dateUnix) - self.assertEqual(matchlog.group(), 'Jan 23 21:59:59') + self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand @@ -150,7 +152,7 @@ class DateDetectorTest(LogCaptureTestCase): ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) - self.assertEqual(logMatch.group(), rdate) + self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) @@ -164,7 +166,7 @@ class DateDetectorTest(LogCaptureTestCase): ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) - self.assertEqual(logMatch.group(), rdate) + self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) @@ -183,23 +185,23 @@ class DateDetectorTest(LogCaptureTestCase): self.assertNotEqual(logdate, None) ( logTime, logMatch ) = logdate self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') + self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end for i in xrange(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '11/10/2012 02:37:17') + self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') # and now back to the original ( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') + self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') def testDateTemplate(self): t = DateTemplate() t.setRegex('^a{3,5}b?c*$') - self.assertEqual(t.regex, '^a{3,5}b?c*$') + self.assertEqual(t.regex, '^(a{3,5}b?c*)$') self.assertRaises(Exception, t.getDate, '') - self.assertEqual(t.matchDate('aaaac').group(), 'aaaac') + self.assertEqual(t.matchDate('aaaac').group(1), 'aaaac') ## no word boundaries left and right: t = DatePatternRegex() @@ -208,22 +210,22 @@ class DateDetectorTest(LogCaptureTestCase): self.assertFalse('**' in t.regex) # match date: dt = 'TIME:20050102T010203' - self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt) - self.assertEqual(t.matchDate(dt).group(), dt) + self.assertEqual(t.matchDate('X' + dt + 'X').group(1), dt) + self.assertEqual(t.matchDate(dt).group(1), dt) # wrong year (for exact %ExY): dt = 'TIME:50050102T010203' self.assertFalse(t.matchDate(dt)) - ## start boundary left and word boundary right: + ## start boundary left and word boundary right (automatically if not **): t = DatePatternRegex() - t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS' + t.pattern = '{^LN-BEG}time:%ExY%Exm%ExdT%ExH%ExM%ExS' self.assertTrue('^' in t.regex) # try match date: dt = 'time:20050102T010203' self.assertFalse(t.matchDate('X' + dt)) self.assertFalse(t.matchDate(dt + 'X')) - self.assertEqual(t.matchDate('##' + dt + '...').group(), dt) - self.assertEqual(t.matchDate(dt).group(), dt) + self.assertEqual(t.matchDate('##' + dt + '...').group(1), dt) + self.assertEqual(t.matchDate(dt).group(1), dt) # case sensitive: dt = 'TIME:20050102T010203' self.assertFalse(t.matchDate(dt)) @@ -232,9 +234,9 @@ class DateDetectorTest(LogCaptureTestCase): t = DatePatternRegex() t.pattern = '^%Y %b %d' self.assertTrue('(?iu)' in t.regex) - dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt) - dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt) - dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt) + dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt) def testAmbiguousInOrderedTemplates(self): dd = self.datedetector @@ -259,7 +261,7 @@ class DateDetectorTest(LogCaptureTestCase): logSys.debug('Line: %s', line) match, template = dd.matchTime(line) self.assertTrue(match) - self.assertEqual(match.group(), debit) + self.assertEqual(match.group(1), debit) def testLowLevelLogging(self): # test coverage for the deep (heavy) debug messages: @@ -363,31 +365,30 @@ class CustomDateFormatsTest(unittest.TestCase): ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), - # Explicit bound in start of the line using %ExLB key, + # Explicit bound in start of the line using {^LN-BEG} key, # (negative) in the 1st case without line begin boundary - wrong date may be found, # (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found) # (positive) and in 3th case with line begin boundary - find the correct date ("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), - (None, "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), - ("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), - # Explicit bound in start of the line using %ExLB key, + (None, "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), + ("20031230 010203", "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), + # Explicit bound in start of the line using {^LN-BEG} key, # up to 2 non-alphanumeric chars front, ** - no word boundary on the right - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: dd = defDD else: - dp = DatePatternRegex(dp) dd = DateDetector() dd.appendTemplate(dp) date = dd.getTime(line) if matched: self.assertTrue(date) - self.assertEqual(matched, date[1].group()) + self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(date, None) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index e212ff23..8e25fd77 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -743,6 +743,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "maxretry = 3", "findtime = 10m", "failregex = ^\s*failure (401|403) from ", + "datepattern = {^LN-BEG}EPOCH", "", "[test-jail1]", "backend = " + backend, "filter =", "action = ", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index c62377a9..5767a3f7 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -101,6 +101,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWrongIngnoreRE(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "{^LN-BEG}EPOCH", "test", r".*? from $", r".**" ) self.assertFalse(fail2banRegex.start(opts, args)) @@ -108,6 +109,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectFound(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--print-no-missed", "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", r"Authentication failure for .*? from $" @@ -136,6 +138,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -151,6 +154,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -160,6 +164,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw_noDns(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", "--usedns=no", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -169,6 +174,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_2(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 @@ -178,6 +184,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testVerbose(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--verbose", "--verbose-date", "--print-no-missed", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 @@ -190,6 +197,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) self.assertTrue(fail2banRegex.start(opts, args)) @@ -203,6 +211,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronCharDebuggex(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 6e7bf367..ed68e2a8 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -36,6 +36,7 @@ class AddFailure(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(AddFailure, self).setUp() self.__items = None self.__failManager = FailManager() diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index 51d3974c..2044c387 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (using precise year pattern): -# failJSON: { "match": false} +# -- wrong time direct in journal-line (used last known date): +# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (using precise year pattern): -# failJSON: { "match": false} +# -- wrong time direct in journal-line (used last known date): +# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" } diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ea996072..227050f2 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -270,6 +270,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p class BasicFilter(unittest.TestCase): def setUp(self): + super(BasicFilter, self).setUp() self.filter = Filter('name') def testGetSetUseDNS(self): @@ -363,6 +364,7 @@ class IgnoreIP(LogCaptureTestCase): setUpMyTime() self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addFailRegex('') + self.filter.setDatePattern('{^LN-BEG}EPOCH') self.filter.processLineAndAdd('1387203300.222 192.168.1.32') self.assertLogged('Ignore 192.168.1.32') tearDownMyTime() @@ -461,6 +463,7 @@ class LogFileFilterPoll(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(LogFileFilterPoll, self).setUp() self.filter = FilterPoll(DummyJail()) self.filter.addLogPath(LogFileFilterPoll.FILENAME) @@ -653,6 +656,8 @@ class LogFileMonitor(LogCaptureTestCase): self.assertLogged('Unable to open %s' % self.name) def testErrorProcessLine(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.filter.sleeptime /= 1000.0 ## produce error with not callable processLine: _org_processLine = self.filter.processLine @@ -715,6 +720,8 @@ class LogFileMonitor(LogCaptureTestCase): pass def testNewChangeViaGetFailures_simple(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # suck in lines from this sample log file self.filter.getFailures(self.name) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) @@ -730,6 +737,8 @@ class LogFileMonitor(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_rewrite(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we rewrite the file at once self.file.close() @@ -748,6 +757,8 @@ class LogFileMonitor(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_move(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we move file into a new location while it has been open already self.file.close() @@ -769,6 +780,7 @@ class CommonMonitorTestCase(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(CommonMonitorTestCase, self).setUp() self._failTotal = 0 def waitFailTotal(self, count, delay=1.): @@ -819,6 +831,8 @@ def get_monitor_failures_testcase(Filter_): self.jail = DummyJail() self.filter = Filter_(self.jail) self.filter.addLogPath(self.name, autoSeek=False) + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") self.filter.start() @@ -1223,6 +1237,8 @@ class GetFailures(LogCaptureTestCase): self.jail = DummyJail() self.filter = FileFilter(self.jail) self.filter.active = True + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # TODO Test this #self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") #self.filter.setTimePattern("%b %d %H:%M:%S") @@ -1329,6 +1345,11 @@ class GetFailures(LogCaptureTestCase): output = (('212.41.96.186', 4, 1124013600.0), ('212.41.96.185', 2, 1124013598.0)) + # speedup search using exact date pattern: + self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', + '^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', + '^EPOCH' + )) self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0) self.filter.addFailRegex("Invalid user .* ") @@ -1358,6 +1379,8 @@ class GetFailures(LogCaptureTestCase): if enc is not None: self.tearDown();self.setUp(); self.filter.setLogEncoding(enc); + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) @@ -1533,6 +1556,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(DNSUtilsNetworkTests, self).setUp() unittest.F2B.SkipIfNoNetwork() def test_IPAddr(self): diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 908e4f6c..cd841a0f 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -86,6 +86,7 @@ def _getSysPythonVersion(): class SetupTest(unittest.TestCase): def setUp(self): + super(SetupTest, self).setUp() unittest.F2B.SkipIfFast() setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 31b1812e..0ed3e764 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -43,6 +43,7 @@ class FilterSamplesRegex(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(FilterSamplesRegex, self).setUp() self.filter = Filter(None) self.filter.active = True diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index cd7ce5c9..03fd1a58 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -65,7 +65,7 @@ class TransmitterBase(unittest.TestCase): def setUp(self): """Call before every test case.""" - #super(TransmitterBase, self).setUp() + super(TransmitterBase, self).setUp() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -301,9 +301,11 @@ class Transmitter(TransmitterBase): ("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"), jail=self.jailName) self.setGetTest( - "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) + "datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName) self.setGetTest( - "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) + "datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName) + self.setGetTest( + "datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self): diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 5bf0be57..1a94a952 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -41,6 +41,7 @@ class Socket(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(Socket, self).setUp() self.server = AsyncServer(self) sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'socket') os.close(sock_fd) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 22079456..fc60f87b 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -526,6 +526,16 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'): self.fail(msg) unittest.TestCase.assertNotIn = assertNotIn +_org_setUp = unittest.TestCase.setUp +def _customSetUp(self): + # print('=='*10, self) + if unittest.F2B.log_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! + print("") + logSys.debug('='*10 + ' %s ' + '='*20, self.id()) + _org_setUp(self) + +unittest.TestCase.setUp = _customSetUp + class LogCaptureTestCase(unittest.TestCase): @@ -601,12 +611,11 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy) logSys.handlers = [self._log] - if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! - print("") + if self._old_level <= logging.DEBUG: logSys.handlers += self._old_handlers - logSys.debug('='*10 + ' %s ' + '='*20, self.id()) - else: + else: # lowest log level to capture messages logSys.setLevel(logging.DEBUG) + super(LogCaptureTestCase, self).setUp() def tearDown(self): """Call after every test case.""" @@ -615,6 +624,7 @@ class LogCaptureTestCase(unittest.TestCase): logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level + super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): logged = self._log.getvalue() From faee5f1fdc4a91dd39dff230ee0aea2d0fefcf70 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 Oct 2016 18:33:01 +0200 Subject: [PATCH 16/91] better caching (thereby better performance), better recognition of similar regex --- config/filter.d/ejabberd-auth.conf | 5 +- fail2ban/server/datedetector.py | 101 ++++++++++++++++++----------- fail2ban/server/datetemplate.py | 5 +- fail2ban/tests/servertestcase.py | 8 +-- 4 files changed, 73 insertions(+), 46 deletions(-) diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf index edd87463..56517489 100644 --- a/config/filter.d/ejabberd-auth.conf +++ b/config/filter.d/ejabberd-auth.conf @@ -34,4 +34,7 @@ maxlines = 2 # journalmatch = -datepattern = ^(?:=[^=]+={3,} )?({DATE}) +#datepattern = ^(?:=[^=]+={3,} )?({DATE}) +# explicit time format using prefix =...==== and no date in second string begins with I(...)... +datepattern = ^(?:=[^=]+={3,} )?(%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?) + ^I\(()** diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index cced6d91..458f8d11 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -36,6 +36,46 @@ logSys = getLogger(__name__) logLevel = 6 RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE) +DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60) + + +def _getPatternTemplate(pattern, key=None): + if key is None: + key = pattern + if '%' not in pattern: + key = pattern.upper() + template = DD_patternCache.get(key) + + if not template: + if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"): + template = DateEpoch(lineBeginOnly=(key != "EPOCH")) + elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"): + template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False)) + else: + template = DatePatternRegex(pattern) + + DD_patternCache.set(key, template) + return template + +def _getAnchoredTemplate(template, wrap=lambda s: '{^LN-BEG}' + s): + # wrap name: + name = wrap(template.name) + # try to find in cache (by name): + template2 = DD_patternCache.get(name) + if not template2: + # wrap pattern (or regexp if not pattern template): + regex = wrap(getattr(template, 'pattern', template.regex)) + if hasattr(template, 'pattern'): + # try to find in cache (by pattern): + template2 = DD_patternCache.get(regex) + # make duplicate and set new anchored regex: + if not template2: + if not hasattr(template, 'pattern'): + template2 = _getPatternTemplate(name) + else: + template2 = _getPatternTemplate(regex) + return template2 + class DateDetectorCache(object): @@ -57,22 +97,20 @@ class DateDetectorCache(object): self._addDefaultTemplate() return self.__templates - def _cacheTemplate(self, template, lineBeginOnly=False): + def _cacheTemplate(self, template): """Cache Fail2Ban's default template. """ if isinstance(template, str): - # exact given template with word benin-end boundary: - if not lineBeginOnly: - template = DatePatternRegex(template) - else: - template = DatePatternRegex(template, wordBegin='start') - # additional template, that prefers datetime at start of a line (safety+performance feature): - if not lineBeginOnly and hasattr(template, 'regex'): - template2 = copy.copy(template) - regex = getattr(template, 'pattern', template.regex) - template2.setRegex(regex, wordBegin='start', wordEnd=True) - if template2.name != template.name: + # exact given template with word begin-end boundary: + template = _getPatternTemplate(template) + # if not already line-begin anchored, additional template, that prefers datetime + # at start of a line (safety+performance feature): + name = template.name + if not name.startswith('{^LN-BEG}') and not name.startswith('^') and hasattr(template, 'regex'): + template2 = _getAnchoredTemplate(template) + # prevent to add duplicates: + if template2.name != name: # increase weight of such templates, because they should be always # preferred in template sorting process (bubble up): template2.weight = 100.0 @@ -114,18 +152,17 @@ class DateDetectorCache(object): # with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?") # Epoch - self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True) - self._cacheTemplate(DateEpoch()) + self._cacheTemplate('EPOCH') # Only time information in the log - self._cacheTemplate("%H:%M:%S", lineBeginOnly=True) + self._cacheTemplate("{^LN-BEG}%H:%M:%S") # <09/16/08@05:03:30> - self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True) + self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>") # MySQL: 130322 11:46:11 self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S") # Apache Tomcat self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 - self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True) + self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") # 20050123T215959, 20050123 215959 self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") # prefixed with optional named time zone (monit): @@ -134,7 +171,7 @@ class DateDetectorCache(object): # +00:00 Jan 23 21:59:59.011 2005 self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # TAI64N - self._cacheTemplate(DateTai64n()) + self._cacheTemplate("TAI64N") # self.__templates = self.__tmpcache[0] + self.__tmpcache[1] del self.__tmpcache @@ -171,7 +208,6 @@ class DateDetector(object): templates """ _defCache = DateDetectorCache() - _patternCache = Utils.Cache(maxCount=1000, maxTime=60*60) def __init__(self): self.__templates = list() @@ -215,18 +251,11 @@ class DateDetector(object): key = pattern = template if '%' not in pattern: key = pattern.upper() - template = DateDetector._patternCache.get(key) - + template = DD_patternCache.get(key) if not template: - if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"): - template = DateEpoch(lineBeginOnly=(key != "EPOCH")) - elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"): - template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False)) - elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): + if key in ("{^LN-BEG}", "{DEFAULT}"): flt = \ - lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ - lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ - None + lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else None self.addDefaultTemplate(flt) return elif "{DATE}" in key: @@ -234,9 +263,9 @@ class DateDetector(object): lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) return else: - template = DatePatternRegex(pattern) + template = _getPatternTemplate(pattern, key) - DateDetector._patternCache.set(key, template) + DD_patternCache.set(key, template) self._appendTemplate(template) logSys.info(" date pattern `%r`: `%s`", @@ -253,13 +282,9 @@ class DateDetector(object): if filterTemplate is not None and not filterTemplate(template): continue # if exact pattern available - create copy of template, contains replaced {DATE} with default regex: if preMatch is not None: - deftemplate = template - template = DateDetector._patternCache.get((preMatch, deftemplate.name)) - if not template: - regex = getattr(deftemplate, 'pattern', deftemplate.regex) - template = copy.copy(deftemplate) - template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) - DateDetector._patternCache.set((preMatch, deftemplate.name), template) + # get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}: + template = _getAnchoredTemplate(template, + wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch)) # append date detector template (ignore duplicate if some was added before default): self._appendTemplate(template, ignoreDup=ignoreDup) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 3a9612aa..f670cbbd 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -115,15 +115,14 @@ class DateTemplate(object): self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN if wordBegin != 'start': regex = r'(?:^|\b|\W)' + regex - self.name = '{*WD-BEG}' + self.name else: regex = r"^(?:\W{0,2})?" + regex - self.name = '{^LN-BEG}' + self.name + if not self.name.startswith('{^LN-BEG}'): + self.name = '{^LN-BEG}' + self.name # if word end boundary: if boundEnd: self.flags |= DateTemplate.WORD_END regex += r'(?=\b|\W|$)' - self.name += '{*WD-END}' if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END # remove possible special pattern "**" in front and end of regex: diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 03fd1a58..724e3bb9 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -298,14 +298,14 @@ class Transmitter(TransmitterBase): def testDatePattern(self): self.setGetTest("datepattern", "%%%Y%m%d%H%M%S", - ("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"), + ("%%%Y%m%d%H%M%S", "%YearMonthDay24hourMinuteSecond"), jail=self.jailName) self.setGetTest( - "datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName) + "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) self.setGetTest( - "datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName) + "datepattern", "^Epoch", (None, "{^LN-BEG}Epoch"), jail=self.jailName) self.setGetTest( - "datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName) + "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self): From ffa97054120d487dd5a19b6f88d2ec565c45000b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 17 Oct 2016 12:09:23 +0200 Subject: [PATCH 17/91] fixed UTC/GMT named time zone using `%Z` and `%z` patterns (special case with 0 zone offset); Currently still ignores another named zones, because fail2ban assumes that the given date is in the current default zone. Closes gh-1575 --- fail2ban/server/strptime.py | 8 ++++++-- fail2ban/tests/datedetectortestcase.py | 10 +++++++++- fail2ban/tests/files/logs/zzz-generic-example | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 643984d3..cdfe0e0e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -41,7 +41,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): #timeRE['z'] = r"%s?(?PZ|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] timeRE['Z'] = r"(?P[A-Z]{3,5})" -timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" +timeRE['z'] = r"(?PZ|UTC|GMT|[+-]\d{2}(?::?[0-5]\d)?)" # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: @@ -183,7 +183,7 @@ def reGroupDictStrptime(found_dict, msec=False): week_of_year_start = 0 elif key == 'z': z = val - if z == "Z": + if z in ("Z", "UTC", "GMT"): tzoffset = 0 else: tzoffset = int(z[1:3]) * 60 # Hours... @@ -191,6 +191,10 @@ def reGroupDictStrptime(found_dict, msec=False): tzoffset += int(z[-2:]) # ...and minutes if z.startswith("-"): tzoffset = -tzoffset + elif key == 'Z': + z = val + if z in ("UTC", "GMT"): + tzoffset = 0 # Fail2Ban will assume it's this year assume_year = False diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index fdb7ec71..695f099c 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -378,6 +378,11 @@ class CustomDateFormatsTest(unittest.TestCase): ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), + # UTC/GMT time zone offset (with %z and %Z): + (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03] server ..."), + (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 UTC] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 UTC] server ..."), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: @@ -388,7 +393,10 @@ class CustomDateFormatsTest(unittest.TestCase): date = dd.getTime(line) if matched: self.assertTrue(date) - self.assertEqual(matched, date[1].group(1)) + if isinstance(matched, basestring): + self.assertEqual(matched, date[1].group(1)) + else: + self.assertEqual(matched, date[0]) else: self.assertEqual(date, None) diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index 2044c387..d0c31740 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -46,6 +46,14 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ # failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 +# -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST): +# failJSON: { "time": "2005-06-21T16:55:09", "match": true , "host": "192.0.2.09" } +2005-06-21 16:55:09 machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.09 +# failJSON: { "time": "2005-06-21T16:55:10", "match": true , "host": "192.0.2.10" } +2005-06-21 14:55:10 UTC machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.10 +# failJSON: { "time": "2005-06-21T16:55:11", "match": true , "host": "192.0.2.11" } +2005-06-21 14:55:11 GMT machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.11 + # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" } [Jun 21 16:56:02] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.250 # failJSON: { "match": false, "desc": "test 1st ignoreregex" } From c8b036456dad258a4b9d38168530933bc02cc71c Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 17 Oct 2016 12:11:41 +0200 Subject: [PATCH 18/91] changelog entries --- ChangeLog | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ChangeLog b/ChangeLog index 16c65556..ff728685 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,15 @@ TODO: implementing of options resp. other tasks from PR #1346 ### Fixes * [Grave] memory leak's fixed (gh-1277, gh-1234) +* [Grave] Misleading date patterns defined more precisely (using extended syntax + `%Ex[mdHMS]` for exact two-digit match or e. g. `%ExY` as more precise year + pattern, within same century of last year and the next 3 years) +* [Grave] extends date detector template with distance (position of match in + log-line), to prevent grave collision using (re)ordered template list (e.g. + find-spot of wrong date-match inside foreign input, misleading date patterns + by ambiguous formats, etc.) +* Distance collision check always prefers template with shortest distance + (left for right) if date pattern is not anchored * Tricky bug fix: last position of log file will be never retrieved (gh-795), because of CASCADE all log entries will be deleted from logs table together with jail, if used "INSERT OR REPLACE" statement @@ -28,6 +37,11 @@ TODO: implementing of options resp. other tasks from PR #1346 * Pyinotify-backend: stability fix for sporadically errors in multi-threaded environment (without lock) * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values +* Fixed UTC/GMT named time zone, using `%Z` and `%z` patterns + (special case with 0 zone offset, see gh-1575) +* `filter.d/freeswitch.conf` + - Optional prefixes (server, daemon, dual time) if systemd daemon logs used (gh-1548) + - User part rewritten to accept IPv6 resp. domain after "@" (gh-1548) ### New Features * IPv6 support: @@ -118,6 +132,22 @@ fail2ban-client set loglevel INFO - new replacement for `` in opposition to ``, for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together, without host (dns) +* More precise date template handling (WARNING: theoretically possible incompatibilities): + - datedetector rewritten more strict as earlier; + - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`); + - more as one date pattern can be specified using option `datepattern` now + (new-line separated); + - some default options like `datepattern` can be specified directly in + section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]` + section, because of performance (each extra section costs time); + - option `datepattern` can be specified in jail also (e. g. jails without filters + or custom log-format, new-line separated for multiple patterns); + - if first unnamed group specified in pattern, only this will be cut out from + search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match + pattern, and leaves `date:[] ...` for searching in filter); + - faster match and fewer searching of appropriate templates + (DateDetector.matchTime calls rarer DateTemplate.matchDate now); + - several standard filters extended with exact prefixed or anchored date templates; * fail2ban-testcases: - `assertLogged` extended with parameter wait (to wait up to specified timeout, before we throw assert exception) + test cases rewritten using that From 58717c185484e643b808c3c77feb2520d36e88f3 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 17 Oct 2016 13:26:24 +0200 Subject: [PATCH 19/91] fail2ban-testcases: persistently set (python) time zone to CET during test cases process (used in zone-related test-cases) --- fail2ban/tests/utils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index fc60f87b..ae543e3d 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -260,6 +260,10 @@ def initTests(opts): raise unittest.SkipTest('Skip test because of "--no-network"') unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork + # persistently set time zone to CET (used in zone-related test-cases), + # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match + os.environ['TZ'] = 'Europe/Zurich' + time.tzset() # set alternate now for time related test cases: MyTime.setAlternateNow(TEST_NOW) @@ -292,17 +296,10 @@ old_TZ = os.environ.get('TZ', None) def setUpMyTime(): # Set the time to a fixed, known value # Sun Aug 14 12:00:00 CEST 2005 - # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match - os.environ['TZ'] = 'Europe/Zurich' - time.tzset() MyTime.setTime(TEST_NOW) def tearDownMyTime(): - os.environ.pop('TZ') - if old_TZ: # pragma: no cover - os.environ['TZ'] = old_TZ - time.tzset() MyTime.myTime = None From 4e252be76f1e7a307fb626af9636e745f93547ec Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 25 Oct 2016 11:01:32 +0200 Subject: [PATCH 20/91] Update FILTERS closes #1591 --- FILTERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FILTERS b/FILTERS index 10113dfc..e114973a 100644 --- a/FILTERS +++ b/FILTERS @@ -227,7 +227,7 @@ Regular expressions (failregex, ignoreregex) assume that the date/time has been removed from the log line (this is just how fail2ban works internally ATM). If the format is like ' error 1.2.3.4 is evil' then you need to match -the < at the start so regex should be similar to '^<> is evil$' using +the <> at the start so regex should be similar to '^<> error is evil$' using where the IP/domain name appears in the log line. The following general rules apply to regular expressions: From 8c407665119e37dd3f27433bc20bac813c9d3c05 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 10 Nov 2016 06:29:42 +0100 Subject: [PATCH 21/91] Add Mongodb-auth filter and jail --- config/filter.d/mongodb-auth.conf | 28 ++++++++++++++++++++++++++++ config/jail.conf | 12 ++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 config/filter.d/mongodb-auth.conf diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf new file mode 100644 index 00000000..77443f9c --- /dev/null +++ b/config/filter.d/mongodb-auth.conf @@ -0,0 +1,28 @@ +# Fail2Ban filter for unsuccesfull MongoDB authentication attempts +# +# + +[Init] +maxlines = 10 + +[Definition] +failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]*\s+\[conn(?P=__connid)\] end connection + +ignoreregex = + + +# DEV Notes: +# +# Log example: +# 2016-10-20T09:54:27.108+0200 [initandlisten] connection accepted from 127.0.0.1:53276 #1 (1 connection now open) +# 2016-10-20T09:54:27.109+0200 [conn1] authenticate db: test { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# 2016-10-20T09:54:27.110+0200 [conn1] Failed to authenticate root@test with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@test +# 2016-11-09T09:54:27.894+0100 [conn1] end connection 127.0.0.1:53276 (0 connections now open) +# 2016-11-09T11:55:58.890+0100 [initandlisten] connection accepted from 127.0.0.1:54266 #1510 (1 connection now open) +# 2016-11-09T11:55:58.892+0100 [conn1510] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# 2016-11-09T11:55:58.892+0100 [conn1510] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# 2016-11-09T11:55:58.894+0100 [conn1510] end connection 127.0.0.1:54266 (0 connections now open) +# +# Authors: Alexander Finkhäuser and sebres +# + diff --git a/config/jail.conf b/config/jail.conf index 41d8d3b8..50ecbd72 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,6 +731,18 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +# Log wrong MongoDB auth +# Logfile /var/log/mongodb/mongodb.log +# add setting in /etc/mongodb.conf +# logpath=/var/log/mongodb/mongodb.log +# +# and use of the authentication +# auth = true +[mongodb-auth] +port = 27017 +logpath = /var/log/mongodb/mongodb.log + + # Jail for more extended banning of persistent abusers # !!! WARNINGS !!! # 1. Make sure that your loglevel specified in fail2ban.conf/.local From 8ac28e5dcb73a70cc5caa8ff8df60ed5b901a9d6 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 10 Nov 2016 13:09:32 +0100 Subject: [PATCH 22/91] Make changes and add test file --- config/filter.d/mongodb-auth.conf | 7 +++++++ config/jail.conf | 8 +------- fail2ban/tests/files/logs/mongodb-auth | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 fail2ban/tests/files/logs/mongodb-auth diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf index 77443f9c..43462e73 100644 --- a/config/filter.d/mongodb-auth.conf +++ b/config/filter.d/mongodb-auth.conf @@ -1,5 +1,12 @@ # Fail2Ban filter for unsuccesfull MongoDB authentication attempts # +# Logfile /var/log/mongodb/mongodb.log +# +# add setting in /etc/mongodb.conf +# logpath=/var/log/mongodb/mongodb.log +# +# and use of the authentication +# auth = true # [Init] diff --git a/config/jail.conf b/config/jail.conf index 50ecbd72..0778a318 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,13 +731,7 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s -# Log wrong MongoDB auth -# Logfile /var/log/mongodb/mongodb.log -# add setting in /etc/mongodb.conf -# logpath=/var/log/mongodb/mongodb.log -# -# and use of the authentication -# auth = true +# Log wrong MongoDB auth (for details how ... see filter ...) [mongodb-auth] port = 27017 logpath = /var/log/mongodb/mongodb.log diff --git a/fail2ban/tests/files/logs/mongodb-auth b/fail2ban/tests/files/logs/mongodb-auth new file mode 100644 index 00000000..a9c7b99b --- /dev/null +++ b/fail2ban/tests/files/logs/mongodb-auth @@ -0,0 +1,24 @@ +# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.168.1.35" } +2016-11-20T00:04:00.110+0200 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +2016-11-20T00:04:00.111+0200 [conn1] end connection 192.168.1.35:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.171" } +2016-11-20T00:24:00.110+0200 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +2016-11-20T00:24:00.111+0200 [conn5] end connection 220.95.238.171:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.176" } +2016-11-20T00:24:00.110+0200 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +2016-11-20T00:24:00.111+0200 [conn334] end connection 220.95.238.176:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "167.96.268.1" } +2016-11-20T00:24:00.110+0200 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +2016-11-20T00:24:00.111+0200 [conn56] end connection 167.96.268.1:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": false , "host": "127.0.0.1" } +2016-11-10T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) +2016-11-10T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) + +# failJSON: { "time": "2016-11-10T13:07:49", "match": false , "host": "177.13.20.178" } +2016-11-10T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +2016-11-10T13:07:49.834+0100 [conn2271] end connection 177.13.20.178:60268 (3 connections now open) + From b5433f48b7b0c7673b43c7956e0b74737b9116ca Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 11 Nov 2016 11:04:10 +0100 Subject: [PATCH 23/91] amend after code review of merge gh-1581 --- ChangeLog | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 10 ++++------ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1ed40dca..ca7864fc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ releases. - recognized "Failed publickey for" (gh-1477); - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) + - optional port part after host (see gh-1533, gh-1581) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index bdc1851e..35cd8754 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -22,7 +22,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ - ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from (?: port \d*)?\s*$ + ^%(__prefix_line)s[iI](?:llegal|nvalid) user .*? from (?: port \d+)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7ceb899c..0800f86b 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -17,8 +17,10 @@ Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 #4 -# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "211.114.51.213" } -Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 211.114.51.213 +# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "192.0.2.1", "desc": "Invalid user" } +Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 192.0.2.1 +# failJSON: { "time": "2005-07-20T14:42:12", "match": true , "host": "192.0.2.2", "desc": "Invalid user with port" } +Jul 20 14:42:12 localhost sshd[22708]: Invalid user ftp from 192.0.2.2 port 37220 #5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch # yoh: added ':' after [sshd] since the case without is not really common any more @@ -167,7 +169,3 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root - -# Match invalid user messages with port at the end -# failJSON: {"time": "2004-10-15T11:35:28", "match": true , "host": "1.2.3.4", "desc": "Invalid user root" } -Oct 15 11:35:28 somehost sshd[7024]: Invalid user root from 1.2.3.4 port 37220 From a2cf34a64ef3348b284135e04659418d2a04ac5f Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 17 Nov 2016 21:04:51 +0100 Subject: [PATCH 24/91] code review: added endpos to found tuple, just to be safe by unpack --- fail2ban/server/datedetector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 458f8d11..01cb43fe 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -359,7 +359,7 @@ class DateDetector(object): if not match: self.__lastTemplIdx = 0x7fffffff logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) - found = None, 0x7fffffff, -1 + found = None, 0x7fffffff, 0x7fffffff, -1 i = 0 for ddtempl in self.__templates: if logSys.getEffectiveLevel() <= logLevel-1: From 189e70d99cc38f09183c26b86cfd880039763314 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 18 Nov 2016 14:42:29 +0100 Subject: [PATCH 25/91] processLine etc. rewritten: - normalize calling parameters (persistent parameters moved from function arguments to filter member variables) - save last line as lambda instead of return it as string (lazy convert of process line tuple to string on demand, needed in fail2ban-regex only) --- fail2ban/client/fail2banregex.py | 6 +++++- fail2ban/server/filter.py | 28 +++++++++++++++++----------- fail2ban/tests/samplestestcase.py | 6 ++++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 4096ac4e..a5aad76f 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -247,6 +247,9 @@ class Fail2banRegex(object): self.raw = True if opts.raw else False if opts.usedns: self._filter.setUseDns(opts.usedns) + self._filter.returnRawHost = self.raw + self._filter.checkFindTime = False + self._filter.checkAllRegex = True def decode_line(self, line): return FileContainer.decode_line('', self.encoding, line) @@ -350,7 +353,8 @@ class Fail2banRegex(object): orgLineBuffer = self._filter._Filter__lineBuffer fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() try: - line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw) + ret = self._filter.processLine(line, date) + line = self._filter.processedLine() for match in ret: # Append True/False flag depending if line was matched by # more than one regex diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9b1aefd6..2487b8f5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -90,6 +90,12 @@ class Filter(JailThread): ## Error counter (protected, so can be used in filter implementations) ## if it reached 100 (at once), run-cycle will go idle self._errors = 0 + ## return raw host (host is not dns): + self.returnRawHost = False + ## check each regex (used for test purposes): + self.checkAllRegex = False + ## if true ignores obsolete failures (failure time < now - findTime): + self.checkFindTime = True ## Ticks counter self.ticks = 0 @@ -455,8 +461,7 @@ class Filter(JailThread): return False - def processLine(self, line, date=None, returnRawHost=False, - checkAllRegex=False, checkFindTime=False): + def processLine(self, line, date=None): """Split the time portion from log msg and return findFailures on them """ if date: @@ -476,14 +481,15 @@ class Filter(JailThread): else: tupleLine = (l, "", "", None) - return "".join(tupleLine[::2]), self.findFailure( - tupleLine, date, returnRawHost, checkAllRegex, checkFindTime) + # save last line (lazy convert of process line tuple to string on demand): + self.processedLine = lambda: "".join(tupleLine[::2]) + return self.findFailure(tupleLine, date) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager """ try: - for element in self.processLine(line, date, checkFindTime=True)[1]: + for element in self.processLine(line, date): ip = element[1] unixTime = element[2] lines = element[3] @@ -539,10 +545,10 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date=None, returnRawHost=False, - checkAllRegex=False, checkFindTime=False): + def findFailure(self, tupleLine, date=None): failList = list() + returnRawHost = self.returnRawHost cidr = IPAddr.CIDR_UNSPEC if self.__useDns == "raw": returnRawHost = True @@ -577,7 +583,7 @@ class Filter(JailThread): timeText = self.__lastTimeText or "".join(tupleLine[::2]) date = self.__lastDate - if checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): + if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): logSys.log(5, "Ignore line since time %s < %s - %s", date, MyTime.time(), self.getFindTime()) return failList @@ -598,7 +604,7 @@ class Filter(JailThread): # The ignoreregex matched. Remove ignored match. self.__lineBuffer = failRegex.getUnmatchedTupleLines() logSys.log(7, "Matched ignoreregex and was ignored") - if not checkAllRegex: + if not self.checkAllRegex: break else: continue @@ -641,7 +647,7 @@ class Filter(JailThread): ip = IPAddr(fid, IPAddr.CIDR_RAW) failList.append([failRegexIndex, ip, date, failRegex.getMatchedLines(), fail]) - if not checkAllRegex: + if not self.checkAllRegex: break else: ips = DNSUtils.textToIp(host, self.__useDns) @@ -649,7 +655,7 @@ class Filter(JailThread): for ip in ips: failList.append([failRegexIndex, ip, date, failRegex.getMatchedLines(), fail]) - if not checkAllRegex: + if not self.checkAllRegex: break except RegexException as e: # pragma: no cover - unsure if reachable logSys.error(e) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 31b1812e..55dffdb0 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -44,6 +44,9 @@ class FilterSamplesRegex(unittest.TestCase): def setUp(self): """Call before every test case.""" self.filter = Filter(None) + self.filter.returnRawHost = True + self.filter.checkAllRegex = True + self.filter.checkFindTime = False self.filter.active = True setUpMyTime() @@ -111,8 +114,7 @@ def testSampleRegexsFactory(name, basedir): else: faildata = {} - ret = self.filter.processLine( - line, returnRawHost=True, checkAllRegex=True)[1] + ret = self.filter.processLine(line) if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True), From c442569b63028c573389a673f7473a4731d43567 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 21 Nov 2016 16:35:33 +0100 Subject: [PATCH 26/91] executeCmd: added possibility to select success return codes ignorecommand: both return codes (0, 1) are success codes now, so no errors will be logged + test cases extended to check this (and error case) --- fail2ban/server/action.py | 4 ++-- fail2ban/server/filter.py | 3 ++- fail2ban/server/utils.py | 12 +++++++----- fail2ban/tests/files/ignorecommand.py | 3 +++ fail2ban/tests/filtertestcase.py | 4 ++++ 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 976adb20..62aa51dc 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -584,7 +584,7 @@ class CommandAction(ActionBase): return self.executeCmd(realCmd, self.timeout) @staticmethod - def executeCmd(realCmd, timeout=60): + def executeCmd(realCmd, timeout=60, **kwargs): """Executes a command. Parameters @@ -613,6 +613,6 @@ class CommandAction(ActionBase): _cmd_lock.acquire() try: - return Utils.executeCmd(realCmd, timeout, shell=True, output=False) + return Utils.executeCmd(realCmd, timeout, shell=True, output=False, **kwargs) finally: _cmd_lock.release() diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 2487b8f5..d263c901 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -455,7 +455,8 @@ class Filter(JailThread): if self.__ignoreCommand: command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } ) logSys.debug('ignore command: ' + command) - ret_ignore = CommandAction.executeCmd(command) + ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1)) + ret_ignore = ret and ret_ignore == 0 self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command") return ret_ignore diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 6ed3d8fa..6d74d2db 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -110,7 +110,7 @@ class Utils(): return flags @staticmethod - def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True): + def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, success_codes=(0,)): """Executes a command. Parameters @@ -178,7 +178,7 @@ class Utils(): if not popen: return False if not output else (False, stdout, stderr, retcode) - std_level = retcode == 0 and logging.DEBUG or logging.ERROR + std_level = logging.DEBUG if retcode in success_codes else logging.ERROR # if we need output (to return or to log it): if output or std_level >= logSys.getEffectiveLevel(): # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode. @@ -208,8 +208,8 @@ class Utils(): popen.stderr.close() success = False - if retcode == 0: - logSys.debug("%-.40s -- returned successfully", realCmd) + if retcode in success_codes: + logSys.debug("%-.40s -- returned successfully %i", realCmd, retcode) success = True elif retcode is None: logSys.error("%-.40s -- unable to kill PID %i", realCmd, popen.pid) @@ -223,7 +223,9 @@ class Utils(): logSys.error("%-.40s -- returned %i", realCmd, retcode) if msg: logSys.info("HINT on %i: %s", retcode, msg % locals()) - return success if not output else (success, stdout, stderr, retcode) + if output: + return success, stdout, stderr, retcode + return success if len(success_codes) == 1 else (success, retcode) @staticmethod def wait_for(cond, timeout, interval=None): diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py index 7011b51b..8c115006 100755 --- a/fail2ban/tests/files/ignorecommand.py +++ b/fail2ban/tests/files/ignorecommand.py @@ -1,5 +1,8 @@ #!/usr/bin/env fail2ban-python import sys +if len(sys.argv) != 2 or sys.argv[1] == "": + sys.stderr.write('usage: ignorecommand IP') + exit(10) if sys.argv[1] == "10.0.0.1": exit(0) exit(1) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2b57ce47..a6f491dd 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -377,6 +377,10 @@ class IgnoreIP(LogCaptureTestCase): self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py ")) self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1")) self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0")) + self.assertLogged("returned successfully 0", "returned successfully 1", all=True) + self.pruneLog() + self.assertFalse(self.filter.inIgnoreIPList("")) + self.assertLogged("usage: ignorecommand IP", "returned 10", all=True) def testIgnoreCauseOK(self): ip = "93.184.216.34" From 701abfd250c7f2e68dd2d6f4490fec624ec8ade5 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 21 Nov 2016 17:00:08 +0100 Subject: [PATCH 27/91] ChangeLog entry added + indentation fix (space-tab replacement) --- ChangeLog | 1 + fail2ban/tests/files/ignorecommand.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6d7ecf42..05a2e38f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ TODO: implementing of options resp. other tasks from PR #1346 * Pyinotify-backend: stability fix for sporadically errors in multi-threaded environment (without lock) * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values +* Misleading errors logged from ignorecommand in success case on retcode 1 (gh-1194) ### New Features * IPv6 support: diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py index 8c115006..da48ff91 100755 --- a/fail2ban/tests/files/ignorecommand.py +++ b/fail2ban/tests/files/ignorecommand.py @@ -1,8 +1,8 @@ #!/usr/bin/env fail2ban-python import sys if len(sys.argv) != 2 or sys.argv[1] == "": - sys.stderr.write('usage: ignorecommand IP') - exit(10) + sys.stderr.write('usage: ignorecommand IP') + exit(10) if sys.argv[1] == "10.0.0.1": exit(0) exit(1) From 261f875748cba31369bdedca5add04059b1c4a5c Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 21 Nov 2016 19:00:53 +0100 Subject: [PATCH 28/91] Fixed sporadic tab-replacement (`\n\t` instead of `\n ` by word wrapping) in mime content of smtp-message in test cases, see https://github.com/fail2ban/fail2ban/pull/1410#issuecomment-262000804 --- fail2ban/tests/action_d/test_smtp.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 5c8b1923..bfbf7b61 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -21,6 +21,7 @@ import os import smtpd import threading import unittest +import re import sys if sys.version_info >= (3, 3): import importlib @@ -38,7 +39,9 @@ class TestSMTPServer(smtpd.SMTPServer): self.peer = peer self.mailfrom = mailfrom self.rcpttos = rcpttos - self.data = data + self.org_data = data + # replace new line (with tab or space) for possible mime translations (word wrap): + self.data = re.sub(r"\n[\t ]", " ", data) class SMTPActionTest(unittest.TestCase): @@ -104,9 +107,9 @@ class SMTPActionTest(unittest.TestCase): self.assertEqual(self.smtpd.rcpttos, ["root"]) subject = "Subject: [Fail2Ban] %s: banned %s" % ( self.jail.name, aInfo['ip']) - self.assertIn(subject, self.smtpd.data.replace("\n", "")) - self.assertTrue( - "%i attempts" % aInfo['failures'] in self.smtpd.data) + self.assertIn(subject, self.smtpd.data) + self.assertIn( + "%i attempts" % aInfo['failures'], self.smtpd.data) self.action.matches = "matches" self.action.ban(aInfo) From f46ada023ee35ebccc38aef2b57992288a276c05 Mon Sep 17 00:00:00 2001 From: Johannes Weberhofer Date: Thu, 10 Mar 2016 15:30:46 +0100 Subject: [PATCH 29/91] Use Fedora's backend-settings for openSUSE Those settings are ok for newer openSUSE versions --- config/paths-opensuse.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/paths-opensuse.conf b/config/paths-opensuse.conf index 0d6ad522..227a5e98 100644 --- a/config/paths-opensuse.conf +++ b/config/paths-opensuse.conf @@ -36,3 +36,15 @@ mysql_log = /var/log/mysql/mysqld.log roundcube_errors_log = /srv/www/roundcubemail/logs/errors solidpop3d_log = %(syslog_mail)s + +# These services will log to the journal via syslog, so use the journal by +# default. +syslog_backend = systemd +sshd_backend = systemd +dropbear_backend = systemd +proftpd_backend = systemd +pureftpd_backend = systemd +wuftpd_backend = systemd +postfix_backend = systemd +dovecot_backend = systemd +mysql_backend = systemd From 528a7a5abbf388def344387cd77f8f2fb54b532d Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 11:14:27 +0100 Subject: [PATCH 30/91] systemd service update: - starting service in normal mode (without forking) - does not restart if service exited normally (exit-code 0, e.g. stopped via fail2ban-client) - does not restart if service can not start (exit-code 255, e.g. wrong configuration, etc.) - service can be additionally started/stopped with commands (fail2ban-client, fail2ban-server) --- files/fail2ban.service | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/files/fail2ban.service b/files/fail2ban.service index e79faed1..5b661100 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -5,12 +5,13 @@ After=network.target iptables.service firewalld.service PartOf=iptables.service firewalld.service [Service] -Type=forking -ExecStart=/usr/bin/fail2ban-client -x start +Type=simple +ExecStart=/usr/bin/fail2ban-server -xf start ExecStop=/usr/bin/fail2ban-client stop ExecReload=/usr/bin/fail2ban-client reload PIDFile=/var/run/fail2ban/fail2ban.pid -Restart=always +Restart=on-failure +RestartPreventExitStatus=0 255 [Install] WantedBy=multi-user.target From 77dc5a334c0e1ba79482a7799fefef27b6df413e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 13:19:09 +0100 Subject: [PATCH 31/91] really skips invalid jails (because of theirs wrong configuration) - server starts nevertheless, as long as one jail was successful configured; message about wrong jail configuration logged in client log (stdout, systemd journal etc.) and in server log as error --- fail2ban/client/jailreader.py | 134 ++++++++++++++++++--------------- fail2ban/client/jailsreader.py | 13 +++- fail2ban/server/transmitter.py | 3 + 3 files changed, 86 insertions(+), 64 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9d01a693..df6c6664 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -118,70 +118,78 @@ class JailReader(ConfigReader): defsec = self._cfg.get_defaults() defsec["fail2ban_version"] = version - # Read first options only needed for merge defaults ('known/...' from filter): - self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) - if not self.__opts: - return False - - if self.isEnabled(): - # Read filter - if self.__opts["filter"]: - filterName, filterOpt = JailReader.extractOptions( - self.__opts["filter"]) - self.__filter = FilterReader( - filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) - ret = self.__filter.read() - # merge options from filter as 'known/...': - self.__filter.getOptions(self.__opts) - ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') - if not ret: - logSys.error("Unable to read the filter") - return False - else: - self.__filter = None - logSys.warning("No filter set for jail %s" % self.__name) + try: - # Read second all options (so variables like %(known/param) can be interpolated): - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + # Read first options only needed for merge defaults ('known/...' from filter): + self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) if not self.__opts: - return False - - # cumulate filter options again (ignore given in jail): - if self.__filter: - self.__filter.getOptions(self.__opts) - - # Read action - for act in self.__opts["action"].split('\n'): - try: - if not act: # skip empty actions - continue - actName, actOpt = JailReader.extractOptions(act) - if actName.endswith(".py"): - self.__actions.append([ - "set", - self.__name, - "addaction", - actOpt.pop("actname", os.path.splitext(actName)[0]), - os.path.join( - self.getBaseDir(), "action.d", actName), - json.dumps(actOpt), - ]) - else: - action = ActionReader( - actName, self.__name, actOpt, - share_config=self.share_config, basedir=self.getBaseDir()) - ret = action.read() - if ret: - action.getOptions(self.__opts) - self.__actions.append(action) + raise ValueError("Init jail options failed") + + if self.isEnabled(): + # Read filter + if self.__opts["filter"]: + filterName, filterOpt = JailReader.extractOptions( + self.__opts["filter"]) + self.__filter = FilterReader( + filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) + ret = self.__filter.read() + # merge options from filter as 'known/...': + self.__filter.getOptions(self.__opts) + ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') + if not ret: + raise ValueError("Unable to read the filter %r" % filterName) + else: + self.__filter = None + logSys.warning("No filter set for jail %s" % self.__name) + + # Read second all options (so variables like %(known/param) can be interpolated): + self.__opts = ConfigReader.getOptions(self, self.__name, opts) + if not self.__opts: + raise ValueError("Read jail options failed") + + # cumulate filter options again (ignore given in jail): + if self.__filter: + self.__filter.getOptions(self.__opts) + + # Read action + for act in self.__opts["action"].split('\n'): + try: + if not act: # skip empty actions + continue + actName, actOpt = JailReader.extractOptions(act) + if actName.endswith(".py"): + self.__actions.append([ + "set", + self.__name, + "addaction", + actOpt.pop("actname", os.path.splitext(actName)[0]), + os.path.join( + self.getBaseDir(), "action.d", actName), + json.dumps(actOpt), + ]) else: - raise AttributeError("Unable to read action") - except Exception as e: - logSys.error("Error in action definition " + act) - logSys.debug("Caught exception: %s" % (e,)) - return False - if not len(self.__actions): - logSys.warning("No actions were defined for %s" % self.__name) + action = ActionReader( + actName, self.__name, actOpt, + share_config=self.share_config, basedir=self.getBaseDir()) + ret = action.read() + if ret: + action.getOptions(self.__opts) + self.__actions.append(action) + else: + raise AttributeError("Unable to read action") + except Exception as e: + logSys.debug("Caught exception: %s" % (e,)) + raise ValueError("Error in action definition %r" % e) + if not len(self.__actions): + logSys.warning("No actions were defined for %s" % self.__name) + + except ValueError as e: + e = str(e) + logSys.error(e) + if not self.__opts: + self.__opts = dict() + self.__opts['config-error'] = e + return False return True def convert(self, allow_no_files=False): @@ -195,6 +203,10 @@ class JailReader(ConfigReader): """ stream = [] + e = self.__opts.get('config-error') + if e: + stream.extend([['config-error', "Jail '%s' skipped, because of wrong configuration: %s" % (self.__name, e)]]) + return stream for opt, value in self.__opts.iteritems(): if opt == "logpath" and \ not self.__opts.get('backend', None).startswith("systemd"): diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 09725ec9..ec7baca7 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -66,7 +66,7 @@ class JailsReader(ConfigReader): sections = [ section ] # Get the options of all jails. - parse_status = True + parse_status = None for sec in sections: if sec == 'INCLUDES': continue @@ -77,11 +77,17 @@ class JailsReader(ConfigReader): ret = jail.getOptions() if ret: if jail.isEnabled(): + # at least one jail was successful: + parse_status = True # We only add enabled jails self.__jails.append(jail) else: logSys.error("Errors in jail %r. Skipping..." % sec) - parse_status = False + self.__jails.append(jail) + if parse_status is None: + parse_status = False + if parse_status is None: + parse_status = True return parse_status def convert(self, allow_no_files=False): @@ -103,7 +109,8 @@ class JailsReader(ConfigReader): stream.extend(jail.convert(allow_no_files=allow_no_files)) # Start jails for jail in self.__jails: - stream.append(["start", jail.getName()]) + if not jail.options.get('config-error'): + stream.append(["start", jail.getName()]) return stream diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2f5be043..ae1017b9 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -131,6 +131,9 @@ class Transmitter: return self.status(command[1:]) elif command[0] == "version": return version.version + elif command[0] == "config-error": + logSys.error(command[1]) + return None raise Exception("Invalid command") def __commandSet(self, command, multiple=False): From c6e8c700f7f712d9b5c93c125bdcefbef670c7ad Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 13:57:06 +0100 Subject: [PATCH 32/91] test cases fixed --- fail2ban/client/jailreader.py | 14 +++++++++----- fail2ban/tests/clientreadertestcase.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index df6c6664..74072481 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -127,9 +127,11 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - if self.__opts["filter"]: - filterName, filterOpt = JailReader.extractOptions( - self.__opts["filter"]) + flt = self.__opts["filter"] + if flt: + filterName, filterOpt = JailReader.extractOptions(flt) + if not filterName: + raise ValueError("Invalid filter declaration %r" % flt) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() @@ -157,6 +159,8 @@ class JailReader(ConfigReader): if not act: # skip empty actions continue actName, actOpt = JailReader.extractOptions(act) + if not actName: + raise ValueError("Invalid action declaration %r" % act) if actName.endswith(".py"): self.__actions.append([ "set", @@ -178,8 +182,8 @@ class JailReader(ConfigReader): else: raise AttributeError("Unable to read action") except Exception as e: - logSys.debug("Caught exception: %s" % (e,)) - raise ValueError("Error in action definition %r" % e) + logSys.debug("Caught exception: %s", e, exc_info=True) + raise ValueError("Error in action definition %r: %r" % (act, e)) if not len(self.__actions): logSys.warning("No actions were defined for %s" % self.__name) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e68523c2..20d6ced1 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -193,13 +193,9 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertLogged('Error in action definition joho[foo') - # This unittest has been deactivated for some time... - # self.assertLogged( - # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0') - # let's test for what is actually logged and handle changes in the future + self.assertLogged("Error in action definition 'joho[foo'") self.assertLogged( - "Caught exception: 'NoneType' object has no attribute 'endswith'") + "Caught exception: Invalid action declaration 'joho[foo'") if STOCK: def testStockSSHJail(self): @@ -496,7 +492,7 @@ class JailsReaderTest(LogCaptureTestCase): def testReadTestJailConf(self): jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jails.read()) - self.assertFalse(jails.getOptions()) + self.assertTrue(jails.getOptions()) self.assertRaises(ValueError, jails.convert) comm_commands = jails.convert(allow_no_files=True) self.maxDiff = None @@ -525,7 +521,12 @@ class JailsReaderTest(LogCaptureTestCase): ['start', 'emptyaction'], ['start', 'missinglogfiles'], ['start', 'brokenaction'], - ['start', 'parse_to_end_of_jail.conf'],])) + ['start', 'parse_to_end_of_jail.conf'], + ['config-error', + 'Jail \'brokenactiondef\' skipped, because of wrong configuration: Error in action definition \'joho[foo\': ValueError("Invalid action declaration \'joho[foo\'",)'], + ['config-error', + "Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"], + ])) self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...") self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction") From e52b47d8f52b9dd1f5704b62f21b3bf9c6c604b5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 13:57:20 +0100 Subject: [PATCH 33/91] normalized log output (all jail parameters in filter are indented with 2 spaces) --- fail2ban/server/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index dcfe09f6..dd4c97c2 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -193,7 +193,7 @@ class Actions(JailThread, Mapping): def setBanTime(self, value): value = MyTime.str2seconds(value) self.__banManager.setBanTime(value) - logSys.info("Set banTime = %s" % value) + logSys.info(" banTime: %s" % value) ## # Get the ban time. From 4882093a41c031d79e3aeace447d762009f6528c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 14:08:35 +0100 Subject: [PATCH 34/91] test cases extended: cover skipping invalid jail --- fail2ban/tests/fail2banclienttestcase.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index e212ff23..cec48cc3 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -768,6 +768,10 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_action_cfg(actname="test-action2") _write_jail_cfg(enabled=[1], actions=[1,2]) + # append one wrong configured jail: + _write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", + "", "filter = broken-jail-filter", "enabled = true") + _write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3)) _write_file(test2log, "w") _write_file(test3log, "w") @@ -786,6 +790,12 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "stdout: '[test-jail1] test-action1: ** start'", "stdout: '[test-jail1] test-action2: ** start'", all=True) + + # broken jail was logged (in client and server log): + self.assertLogged( + "Unable to read the filter 'broken-jail-filter'", + "Errors in jail 'broken-jail'. Skipping...", + "Jail 'broken-jail' skipped, because of wrong configuration", all=True) # enable both jails, 3 logs for jail1, etc... # truncate test-log - we should not find unban/ban again by reload: From 3e9852d4d2a7c73936a4cbaafd17bf13a0e412a6 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 14:56:54 +0100 Subject: [PATCH 35/91] code review, increase coverage --- fail2ban/client/jailreader.py | 24 +++++++++++++++--------- fail2ban/client/jailsreader.py | 10 ++-------- fail2ban/tests/clientreadertestcase.py | 18 ++++++++++++++---- fail2ban/tests/config/jail.conf | 8 ++++++++ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 74072481..b63df5f1 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -122,8 +122,8 @@ class JailReader(ConfigReader): # Read first options only needed for merge defaults ('known/...' from filter): self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) - if not self.__opts: - raise ValueError("Init jail options failed") + if not self.__opts: # pragma: no cover + raise JailDefError("Init jail options failed") if self.isEnabled(): # Read filter @@ -131,7 +131,7 @@ class JailReader(ConfigReader): if flt: filterName, filterOpt = JailReader.extractOptions(flt) if not filterName: - raise ValueError("Invalid filter declaration %r" % flt) + raise JailDefError("Invalid filter definition %r" % flt) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() @@ -139,15 +139,15 @@ class JailReader(ConfigReader): self.__filter.getOptions(self.__opts) ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') if not ret: - raise ValueError("Unable to read the filter %r" % filterName) + raise JailDefError("Unable to read the filter %r" % filterName) else: self.__filter = None logSys.warning("No filter set for jail %s" % self.__name) # Read second all options (so variables like %(known/param) can be interpolated): self.__opts = ConfigReader.getOptions(self, self.__name, opts) - if not self.__opts: - raise ValueError("Read jail options failed") + if not self.__opts: # pragma: no cover + raise JailDefError("Read jail options failed") # cumulate filter options again (ignore given in jail): if self.__filter: @@ -160,7 +160,7 @@ class JailReader(ConfigReader): continue actName, actOpt = JailReader.extractOptions(act) if not actName: - raise ValueError("Invalid action declaration %r" % act) + raise JailDefError("Invalid action definition %r" % act) if actName.endswith(".py"): self.__actions.append([ "set", @@ -180,14 +180,16 @@ class JailReader(ConfigReader): action.getOptions(self.__opts) self.__actions.append(action) else: - raise AttributeError("Unable to read action") + raise JailDefError("Unable to read action %r" % actName) + except JailDefError: + raise except Exception as e: logSys.debug("Caught exception: %s", e, exc_info=True) raise ValueError("Error in action definition %r: %r" % (act, e)) if not len(self.__actions): logSys.warning("No actions were defined for %s" % self.__name) - except ValueError as e: + except JailDefError as e: e = str(e) logSys.error(e) if not self.__opts: @@ -280,3 +282,7 @@ class JailReader(ConfigReader): val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() return option_name, option_opts + + +class JailDefError(Exception): + pass diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index ec7baca7..7d81d0a0 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -84,11 +84,8 @@ class JailsReader(ConfigReader): else: logSys.error("Errors in jail %r. Skipping..." % sec) self.__jails.append(jail) - if parse_status is None: - parse_status = False - if parse_status is None: - parse_status = True - return parse_status + if parse_status is None: parse_status = False + return True if parse_status != False else False def convert(self, allow_no_files=False): """Convert read before __opts and jails to the commands stream @@ -101,9 +98,6 @@ class JailsReader(ConfigReader): """ stream = list() - for opt in self.__opts: - if opt == "": - stream.append([]) # Convert jails for jail in self.__jails: stream.extend(jail.convert(allow_no_files=allow_no_files)) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 20d6ced1..33c217cd 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -193,9 +193,15 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertLogged("Error in action definition 'joho[foo'") - self.assertLogged( - "Caught exception: Invalid action declaration 'joho[foo'") + self.assertLogged("Invalid action definition 'joho[foo'") + + def testJailFilterBrokenDef(self): + jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG) + self.assertTrue(jail.read()) + self.assertFalse(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + self.assertLogged("Invalid filter definition 'flt[test'") if STOCK: def testStockSSHJail(self): @@ -523,7 +529,11 @@ class JailsReaderTest(LogCaptureTestCase): ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'], ['config-error', - 'Jail \'brokenactiondef\' skipped, because of wrong configuration: Error in action definition \'joho[foo\': ValueError("Invalid action declaration \'joho[foo\'",)'], + "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], + ['config-error', + "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"], + ['config-error', + "Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"], ['config-error', "Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"], ])) diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index bf1dea45..659e3fd3 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -27,10 +27,18 @@ logpath = /weapons/of/mass/destruction enabled = true action = joho[foo +[brokenfilterdef] +enabled = true +filter = flt[test + [brokenaction] enabled = true action = brokenaction +[missingaction] +enabled = true +action = noactionfileforthisaction + [missingbitsjail] enabled = true filter = catchallthebadies From fdac44ca589ff5d9de8afde3865022dc7f702858 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 17:08:44 +0100 Subject: [PATCH 36/91] introduced new option `-t` or `--test` to test configuration resp. start server only if configuration is clean (not skip wrong configured jails if option `-t` specified); --- fail2ban/client/configurator.py | 4 +-- fail2ban/client/fail2bancmdline.py | 33 ++++++++++++++++---- fail2ban/client/fail2banserver.py | 38 ++++++++++++------------ fail2ban/client/jailsreader.py | 13 ++++---- fail2ban/tests/fail2banclienttestcase.py | 30 +++++++++++++++++++ 5 files changed, 86 insertions(+), 32 deletions(-) diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 4d28238f..e8472ac1 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -72,9 +72,9 @@ class Configurator: def getEarlyOptions(self): return self.__fail2ban.getEarlyOptions() - def getOptions(self, jail=None, updateMainOpt=None): + def getOptions(self, jail=None, updateMainOpt=None, ignoreWrong=True): self.__fail2ban.getOptions(updateMainOpt) - return self.__jails.getOptions(jail) + return self.__jails.getOptions(jail, ignoreWrong=ignoreWrong) def convertToProtocol(self): self.__streams["general"] = self.__fail2ban.convert() diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 74236ab1..7d0eeead 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -47,6 +47,7 @@ class Fail2banCmdLine(): def __init__(self): self._argv = self._args = None self._configurator = None + self.cleanConfOnly = False self.resetConf() def resetConf(self): @@ -101,6 +102,7 @@ class Fail2banCmdLine(): output(" --logtarget |STDOUT|STDERR|SYSLOG") output(" --syslogsocket auto|") output(" -d dump configuration. For debugging") + output(" -t, --test test configuration (can be also specified with start parameters)") output(" -i interactive mode") output(" -v increase verbosity") output(" -q decrease verbosity") @@ -136,6 +138,9 @@ class Fail2banCmdLine(): self._conf[ o[2:] ] = opt[1] elif o == "-d": self._conf["dump"] = True + elif o == "-t" or o == "--test": + self.cleanConfOnly = True + self._conf["test"] = True elif o == "-v": self._conf["verbose"] += 1 elif o == "-q": @@ -173,8 +178,8 @@ class Fail2banCmdLine(): # Reads the command line options. try: - cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'timeout=', 'help', 'version'] + cmdOpts = 'hc:s:p:xfbdtviqV' + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'timeout=', 'help', 'version'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -225,13 +230,30 @@ class Fail2banCmdLine(): logSys.info("Using pid file %s, [%s] logging to %s", self._conf["pidfile"], logging.getLevelName(llev), self._conf["logtarget"]) + readcfg = True if self._conf.get("dump", False): - ret, stream = self.readConfig() + if readcfg: + ret, stream = self.readConfig() + readcfg = False self.dumpConfig(stream) - return ret + if not self._conf.get("test", False): + return ret + + if self._conf.get("test", False): + if readcfg: + readcfg = False + ret, stream = self.readConfig() + if not ret: + raise ServerExecutionException("ERROR: test configuration failed") + # exit after test if no commands specified (test only): + if not len(self._args): + output("OK: configuration test is successful") + return ret # Nothing to do here, process in client/server return None + except ServerExecutionException: + raise except Exception as e: output("ERROR: %s" % (e,)) if verbose > 2: @@ -246,7 +268,8 @@ class Fail2banCmdLine(): try: self.configurator.Reload() self.configurator.readAll() - ret = self.configurator.getOptions(jail, self._conf) + ret = self.configurator.getOptions(jail, self._conf, + ignoreWrong=not self.cleanConfOnly) self.configurator.convertToProtocol() stream = self.configurator.getConfigStream() except Exception as e: diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index dfee34d2..006a02cf 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -144,27 +144,27 @@ class Fail2banServer(Fail2banCmdLine): return cli def start(self, argv): - # Command line options - ret = self.initCmdLine(argv) - if ret is not None: - return ret - - # Commands - args = self._args - - cli = None - # Just start: - if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False): - pass - else: - # If client mode - whole processing over client: - if len(args) or self._conf.get("interactive", False): - cli = self._Fail2banClient() - return cli.start(argv) - - # Start the server: server = None try: + # Command line options + ret = self.initCmdLine(argv) + if ret is not None: + return ret + + # Commands + args = self._args + + cli = None + # Just start: + if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False): + pass + else: + # If client mode - whole processing over client: + if len(args) or self._conf.get("interactive", False): + cli = self._Fail2banClient() + return cli.start(argv) + + # Start the server: from ..server.utils import Utils # background = True, if should be new process running in background, otherwise start in foreground # process will be forked in daemonize, inside of Server module. diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 7d81d0a0..cd3409b4 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -54,7 +54,7 @@ class JailsReader(ConfigReader): self.__jails = list() return ConfigReader.read(self, "jail") - def getOptions(self, section=None): + def getOptions(self, section=None, ignoreWrong=True): """Reads configuration for jail(s) and adds enabled jails to __jails """ opts = [] @@ -66,7 +66,7 @@ class JailsReader(ConfigReader): sections = [ section ] # Get the options of all jails. - parse_status = None + parse_status = 0 for sec in sections: if sec == 'INCLUDES': continue @@ -78,14 +78,15 @@ class JailsReader(ConfigReader): if ret: if jail.isEnabled(): # at least one jail was successful: - parse_status = True + parse_status |= 1 # We only add enabled jails self.__jails.append(jail) else: - logSys.error("Errors in jail %r. Skipping..." % sec) + logSys.error("Errors in jail %r.%s", sec, " Skipping..." if ignoreWrong else "") self.__jails.append(jail) - if parse_status is None: parse_status = False - return True if parse_status != False else False + # at least one jail was invalid: + parse_status |= 2 + return ((ignoreWrong and parse_status & 1) or not (parse_status & 2)) def convert(self, allow_no_files=False): """Convert read before __opts and jails to the commands stream diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index cec48cc3..e68d9779 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -675,6 +675,36 @@ class Fail2banServerTest(Fail2banClientServerBase): self.pruneLog() os.remove(pjoin(tmp, "f2b.sock")) + @with_tmpdir + @with_kill_srv + def testServerTestFailStart(self, tmp): + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + cfg = pjoin(tmp, "config") + + # test configuration is correct: + self.pruneLog("[test-phase 0]") + self.execSuccess(startparams, "--test") + self.assertLogged("OK: configuration test is successful") + + # append one wrong configured jail: + _write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", + "", "filter = broken-jail-filter", "enabled = true") + + # first try test config: + self.pruneLog("[test-phase 0a]") + self.execFailed(startparams, "--test") + self.assertLogged("Unable to read the filter 'broken-jail-filter'", + "Errors in jail 'broken-jail'.", + "ERROR: test configuration failed", all=True) + + # failed to start with test config: + self.pruneLog("[test-phase 0b]") + self.execFailed(startparams, "-t", "start") + self.assertLogged("Unable to read the filter 'broken-jail-filter'", + "Errors in jail 'broken-jail'.", + "ERROR: test configuration failed", all=True) + @with_tmpdir def testKillAfterStart(self, tmp): try: From 8ed5b44bfd2256f8ffe8ad867b7cd9fc7cc0e467 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 17:38:32 +0100 Subject: [PATCH 37/91] no cover for sporadic executed (time-related) code pieces (just to prevent randomly increasing/decreasing of coverage) --- fail2ban/client/fail2banclient.py | 2 +- fail2ban/server/asyncserver.py | 2 +- fail2ban/server/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 007afd57..0a1ae4f1 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -125,7 +125,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): if client: try : client.close() - except Exception as e: + except Exception as e: # pragma: no cover if showRet or self._conf["verbose"] > 1: logSys.debug(e) if showRet or c[0] == 'echo': diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index d1818d7a..9cc74658 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -241,7 +241,7 @@ class AsyncServer(asyncore.dispatcher): def _remove_sock(self): try: os.remove(self.__sock) - except OSError as e: + except OSError as e: # pragma: no cover if e.errno != errno.ENOENT: raise diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 6d74d2db..57da495a 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -170,7 +170,7 @@ class Utils(): time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) retcode = popen.poll() #logSys.debug("%s -- killed %s ", realCmd, retcode) - if retcode is None and not Utils.pid_exists(pgid): + if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover retcode = signal.SIGKILL except OSError as e: stderr = "%s -- failed with %s" % (realCmd, e) From 7256a5cb8e0c37bca69fd684bd3e568a4cc46540 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Nov 2016 17:55:27 +0100 Subject: [PATCH 38/91] code review: back to previous code - no skipping in testReadTestJailConf --- fail2ban/tests/clientreadertestcase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 33c217cd..5129df61 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -498,7 +498,7 @@ class JailsReaderTest(LogCaptureTestCase): def testReadTestJailConf(self): jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) self.assertTrue(jails.read()) - self.assertTrue(jails.getOptions()) + self.assertFalse(jails.getOptions(ignoreWrong=False)) self.assertRaises(ValueError, jails.convert) comm_commands = jails.convert(allow_no_files=True) self.maxDiff = None @@ -537,7 +537,8 @@ class JailsReaderTest(LogCaptureTestCase): ['config-error', "Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"], ])) - self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...") + self.assertLogged("Errors in jail 'missingbitsjail'.") + self.assertNotLogged("Skipping...") self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction") if STOCK: From 1cd67ecaa260bdaa6e62383ed783d5910587c258 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 23 Nov 2016 22:03:12 +0100 Subject: [PATCH 39/91] automatically creates /var/run/fail2ban before start fail2ban (systems which /var/run/ is virtual resp. memory mount device) --- files/fail2ban.service | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service b/files/fail2ban.service index 5b661100..1ef7b5e6 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service [Service] Type=simple +ExecStartPre=/bin/mkdir -p /var/run/fail2ban ExecStart=/usr/bin/fail2ban-server -xf start ExecStop=/usr/bin/fail2ban-client stop ExecReload=/usr/bin/fail2ban-client reload From 45174c5eaf1479a2b5fec594dce421364b5cfd36 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 24 Nov 2016 12:13:47 +0100 Subject: [PATCH 40/91] if fail2ban running as systemd-service, for logging to the systemd-journal, the `logtarget` could be set to STDOUT small fixes by logging in stdout (+ system targets also allowed in lowercase now) --- fail2ban/client/fail2bancmdline.py | 1 + fail2ban/server/server.py | 10 ++++++---- files/fail2ban.service | 2 ++ man/fail2ban-server.1 | 3 +++ man/jail.conf.5 | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 74236ab1..c7c40120 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -274,6 +274,7 @@ class Fail2banCmdLine(): def exit(code=0): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: + logging.shutdown() sys.stdout.flush() sys.stderr.flush() # exit diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index efd47b09..313b6ee5 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -530,17 +530,19 @@ class Server: # @param target the logging target def setLogTarget(self, target): + # check reserved targets in uppercase, don't change target, because it can be file: + systarget = target.upper() with self.__loggingLock: # don't set new handlers if already the same # or if "INHERITED" (foreground worker of the test cases, to prevent stop logging): if self.__logTarget == target: return True - if target == "INHERITED": + if systarget == "INHERITED": self.__logTarget = target return True # set a format which is simpler for console use fmt = "%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s" - if target == "SYSLOG": + if systarget == "SYSLOG": # Syslog daemons already add date to the message. fmt = "%(name)s[%(process)d]: %(levelname)s %(message)s" facility = logging.handlers.SysLogHandler.LOG_DAEMON @@ -559,9 +561,9 @@ class Server: "Syslog socket file: %s does not exists" " or is not a socket" % self.__syslogSocket) return False - elif target == "STDOUT": + elif systarget == "STDOUT": hdlr = logging.StreamHandler(sys.stdout) - elif target == "STDERR": + elif systarget == "STDERR": hdlr = logging.StreamHandler(sys.stderr) else: # Target should be a file diff --git a/files/fail2ban.service b/files/fail2ban.service index 1ef7b5e6..6eeba957 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -8,6 +8,8 @@ PartOf=iptables.service firewalld.service Type=simple ExecStartPre=/bin/mkdir -p /var/run/fail2ban ExecStart=/usr/bin/fail2ban-server -xf start +# if should be logged in systemd journal, use following line or set logtarget to stdout in fail2ban.local +# ExecStart=/usr/bin/fail2ban-server -xf --logtarget=stdout start ExecStop=/usr/bin/fail2ban-client stop ExecReload=/usr/bin/fail2ban-client reload PIDFile=/var/run/fail2ban/fail2ban.pid diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index de8ba6a4..09dcd65a 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -23,6 +23,9 @@ pidfile path logging level .HP \fB\-\-logtarget\fR |STDOUT|STDERR|SYSLOG +logging target +.br +Note. If fail2ban running as systemd-service, for logging to the systemd-journal, the logtarget could be set to STDOUT .HP \fB\-\-syslogsocket\fR auto| .TP diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 51a00cdc..2e333e5a 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -130,7 +130,9 @@ The items that can be set are: verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) .TP .B logtarget -log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR +log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local +.br +Note. If fail2ban running as systemd-service, for logging to the systemd-journal, the logtarget could be set to STDOUT .br Only a single log target can be specified. If you change logtarget from the default value and you are using logrotate -- also adjust or disable rotation in the From 4f5389fee5ae97630a6f68d2823378011e66345d Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 24 Nov 2016 19:30:10 +0100 Subject: [PATCH 41/91] Update jail.conf --- config/jail.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 41d8d3b8..7200f4eb 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -810,8 +810,9 @@ maxretry = 1 [pass2allow-ftp] # this pass2allow example allows FTP traffic after successful HTTP authentication port = ftp,ftp-data,ftps,ftps-data -# knocking_url variable must be overridden to some secret value in filter.d/apache-pass.local -filter = apache-pass +# knocking_url variable must be overridden to some secret value in jail.local +knocking_url = /knocking/ +filter = apache-pass[knocking_url="%(knocking_url)s"] # access log of the website with HTTP auth logpath = %(apache_access_log)s blocktype = RETURN From d908688b565fbc55fd63c1cf4ac305379afdfdca Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 24 Nov 2016 20:25:08 +0100 Subject: [PATCH 42/91] ChangeLog update --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 05a2e38f..ef2ce050 100644 --- a/ChangeLog +++ b/ChangeLog @@ -55,6 +55,8 @@ TODO: implementing of options resp. other tasks from PR #1346 banned in this jail, if option `--unban` specified - `unban --all` - unbans all IP addresses (in all jails and database) - `unban ... ` - unbans \ (in all jails and database) (see gh-1388) + - introduced new option `-t` or `--test` to test configuration resp. start server only + if configuration is clean (fails by wrong configured jails if option `-t` specified) * New command action parameter `actionrepair` - command executed in order to restore sane environment in error case of `actioncheck`. @@ -119,6 +121,10 @@ fail2ban-client set loglevel INFO - new replacement for `` in opposition to ``, for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together, without host (dns) +* Misconfigured jails don't prevent fail2ban from starting, server starts + nevertheless, as long as one jail was successful configured (gh-1619) + Message about wrong jail configuration logged in client log (stdout, systemd + journal etc.) and in server log with error level * fail2ban-testcases: - `assertLogged` extended with parameter wait (to wait up to specified timeout, before we throw assert exception) + test cases rewritten using that From 308bba448ca7edfd46115b16d27ce18b2f2999c9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 24 Nov 2016 20:43:55 +0100 Subject: [PATCH 43/91] ChangeLog update --- ChangeLog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index 05a2e38f..b07d2842 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,16 @@ TODO: implementing of options resp. other tasks from PR #1346 environment (without lock) * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values * Misleading errors logged from ignorecommand in success case on retcode 1 (gh-1194) +* fail2ban.service - systemd service updated (gh-1618): + - starting service in normal mode (without forking) + - does not restart if service exited normally (exit-code 0, e.g. stopped via fail2ban-client) + - does not restart if service can not start (exit-code 255, e.g. wrong configuration, etc.) + - service can be additionally started/stopped with commands (fail2ban-client, fail2ban-server) + - automatically creates `/var/run/fail2ban` directory before start fail2ban + (systems with virtual resp. memory-based FS for `/var/run`), see gh-1531 + - if fail2ban running as systemd-service, for logging to the systemd-journal, + the `logtarget` could be set to STDOUT + - value `logtarget` for system targets allowed also in lowercase (stdout, stderr, syslog, etc.) ### New Features * IPv6 support: From 097970781cbe61160824c834ca5cb4a065c2a8f8 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 16:53:46 +0100 Subject: [PATCH 44/91] filter/action (and its includes): substitution `%(param)s` may be used now (instead of ``) for init-values specified in jail-configs via `action[param1="...", param2=...]`; substitution `` should be used for dynamic interpolation only (todo: review configurations to replace it); --- fail2ban/client/configparserinc.py | 2 +- fail2ban/client/configreader.py | 32 ++++++++++++++++++++++++------ fail2ban/client/filterreader.py | 6 ++++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index d6cda7f7..35fa7498 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): # pragma: no cover +if sys.version_info >= (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, \ diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 643cdf3a..caab67aa 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -28,13 +28,25 @@ import glob import os from ConfigParser import NoOptionError, NoSectionError -from .configparserinc import SafeConfigParserWithIncludes, logLevel +from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +# if sys.version_info >= (3,5): +# def _merge_dicts(x, y): +# return {**x, **y} +# else: +def _merge_dicts(x, y): + r = x + if y: + r = x.copy() + r.update(y) + return r + + class ConfigReader(): """Generic config reader class. @@ -127,9 +139,9 @@ class ConfigReader(): return self._cfg.options(*args) return {} - def get(self, sec, opt): + def get(self, sec, opt, raw=False, vars={}): if self._cfg is not None: - return self._cfg.get(sec, opt) + return self._cfg.get(sec, opt, raw, vars) return None def getOptions(self, *args, **kwargs): @@ -210,6 +222,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): def getOptions(self, sec, options, pOptions=None, shouldExist=False): values = dict() + if pOptions is None: + pOptions = {} for optname in options: if isinstance(options, (list,tuple)): if len(optname) > 2: @@ -218,15 +232,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): (opttype, optname), optvalue = optname, None else: opttype, optvalue = options[optname] + if optname in pOptions: + continue try: if opttype == "bool": v = self.getboolean(sec, optname) elif opttype == "int": v = self.getint(sec, optname) else: - v = self.get(sec, optname) - if not pOptions is None and optname in pOptions: - continue + v = self.get(sec, optname, vars=pOptions) values[optname] = v except NoSectionError as e: if shouldExist: @@ -289,6 +303,12 @@ class DefinitionInitConfigReader(ConfigReader): return SafeConfigParserWithIncludes.read(self._cfg, self._file) def getOptions(self, pOpts): + # overwrite static definition options with init values, supplied as + # direct parameters from jail-config via action[xtra1="...", xtra2=...]: + if self._initOpts: + if not pOpts: + pOpts = dict() + pOpts = _merge_dicts(pOpts, self._initOpts) self._opts = ConfigReader.getOptions( self, "Definition", self._configOpts, pOpts) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b30f914..5e6b2b74 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import os import shlex -from .configreader import DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader, _merge_dicts from ..server.action import CommandAction from ..helpers import getLogger @@ -50,7 +50,9 @@ class FilterReader(DefinitionInitConfigReader): return self.__file def getCombined(self): - combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) + combinedopts = self._opts + if self._initOpts: + combinedopts = _merge_dicts(self._opts, self._initOpts) if not len(combinedopts): return {} opts = CommandAction.substituteRecursiveTags(combinedopts) From 65abc639cc7838beaf1f587eae74e98afc4a1473 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 16:56:46 +0100 Subject: [PATCH 45/91] allow newline in extra init-parameters of action/filter (or interpolation of it), e. g. action[..., logpath="%(logpath)s"] --- fail2ban/client/jailreader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index b63df5f1..d01064b2 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -43,13 +43,13 @@ logSys = getLogger(__name__) class JailReader(ConfigReader): # regex, to extract list of options: - optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL) # regex, to iterate over single option in option list, syntax: # `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']' # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` optionExtractRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)') + r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) From a2af19c9f0c7ad42e733693f1cbf3d247cded582 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 19:06:34 +0100 Subject: [PATCH 46/91] fixed several actions, that could not work with jails using multiple logpath; additionally repaired execution in default shell (bad substitution by `${x//...}` executing in `/bin/sh`); added helper "action.d/helpers-common.conf", and `_grep_logs` part-command for actions needed grep logs from multiple log-files test cases: executing of some complex actions covered --- config/action.d/complain.conf | 20 ++++- config/action.d/helpers-common.conf | 13 +++ config/action.d/mail-whois-lines.conf | 29 ++++--- config/action.d/sendmail-geoip-lines.conf | 13 +-- config/action.d/sendmail-whois-lines.conf | 13 +-- fail2ban/tests/files/testcase01a.log | 4 + fail2ban/tests/servertestcase.py | 96 +++++++++++++++++++++-- 7 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 config/action.d/helpers-common.conf create mode 100644 fail2ban/tests/files/testcase01a.log diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 9247803e..e4ceb35f 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -28,6 +28,10 @@ # +[INCLUDES] + +before = helpers-common.conf + [Definition] # Option: actionstart @@ -54,10 +58,16 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs} - IP= +actionban = oifs=${IFS}; + IFS=.; SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); + IFS=,; ADDRESSES=$(echo $ADDRESSES) + IFS=${oifs} + IP= if [ ! -z "$ADDRESSES" ]; then - (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])([^0-9]|$)' ) | "Abuse from " ${ADDRESSES//,/\" \"} + ( printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; + printf %%b "\nLines containing failures of (max )\n"; + %(_grep_logs)s; + ) | "Abuse from " $ADDRESSES fi # Option: actionunban @@ -92,3 +102,7 @@ mailcmd = mail -s # mailargs = +# Number of log lines to include in the email +# +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf new file mode 100644 index 00000000..7fa8e9e4 --- /dev/null +++ b/config/action.d/helpers-common.conf @@ -0,0 +1,13 @@ +[DEFAULT] + +# Usage: +# _grep_logs_args = 'test' +# (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... +# +_grep_logs = logpath=""; grep -E %(_grep_logs_args)s $logpath | +_grep_logs_args = '(^|[^0-9])([^0-9]|$)' + +[Init] +greplimit = tail -n +grepmax = 1000 +grepopts = -m \ No newline at end of file diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 6e39c605..cbd970c9 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = mail-whois-common.conf + helpers-common.conf [Definition] @@ -17,7 +18,7 @@ before = mail-whois-common.conf actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban" | -s "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -26,7 +27,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban" | -s "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -40,15 +41,18 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Hi,\n + +_ban_mail_content = ( printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n - `%(_whois_command)s`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Here is more information about :\n" + %(_whois_command)s; + printf %%b "\nLines containing failures of (max )\n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban" ) +actionban = %(_ban_mail_content)s | "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -60,6 +64,12 @@ actionunban = [Init] +# Option: mailcmd +# Notes.: Your system mail command. Is passed 2 args: subject and recipient +# Values: CMD +# +mailcmd = mail -s + # Default name of the chain # name = default @@ -74,4 +84,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index 2232642c..a5616e9f 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + helpers-common.conf [Definition] @@ -19,7 +20,7 @@ before = sendmail-common.conf # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n @@ -33,10 +34,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "" | cut -d':' -f2-` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "" | cut -d':' -f2-` hostname: `host -t A 2>&1`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Lines containing failures of \n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" ) | /usr/sbin/sendmail -f [Init] @@ -50,4 +52,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4156c947..e1c85928 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + helpers-common.conf [Definition] @@ -16,7 +17,7 @@ before = sendmail-common.conf # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n @@ -25,10 +26,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` attempts against .\n\n Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Lines containing failures of \n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" ) | /usr/sbin/sendmail -f [Init] @@ -42,4 +44,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/fail2ban/tests/files/testcase01a.log b/fail2ban/tests/files/testcase01a.log new file mode 100644 index 00000000..203f0517 --- /dev/null +++ b/fail2ban/tests/files/testcase01a.log @@ -0,0 +1,4 @@ +Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:02 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:03 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:04 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 56c85e94..f8915b59 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -28,6 +28,7 @@ import unittest import time import tempfile import os +import re import sys import platform @@ -1609,31 +1610,114 @@ class ServerConfigReaderTests(LogCaptureTestCase): # wrap default command processor: action.executeCmd = self._executeCmd # test start : - logSys.debug('# === start ==='); self.pruneLog() + self.pruneLog('# === start ===') action.start() self.assertLogged(*tests['start'], all=True) # test ban ip4 : - logSys.debug('# === ban-ipv4 ==='); self.pruneLog() + self.pruneLog('# === ban-ipv4 ===') action.ban({'ip': IPAddr('192.0.2.1')}) self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test unban ip4 : - logSys.debug('# === unban ipv4 ==='); self.pruneLog() + self.pruneLog('# === unban ipv4 ===') action.unban({'ip': IPAddr('192.0.2.1')}) self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test ban ip6 : - logSys.debug('# === ban ipv6 ==='); self.pruneLog() + self.pruneLog('# === ban ipv6 ===') action.ban({'ip': IPAddr('2001:DB8::')}) self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test unban ip6 : - logSys.debug('# === unban ipv6 ==='); self.pruneLog() + self.pruneLog('# === unban ipv6 ===') action.unban({'ip': IPAddr('2001:DB8::')}) self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test stop : - logSys.debug('# === stop ==='); self.pruneLog() + self.pruneLog('# === stop ===') action.stop() self.assertLogged(*tests['stop'], all=True) + def _executeMailCmd(self, realCmd, timeout=60): + # replace pipe to mail with pipe to cat: + realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)', + r' echo mail \1 ) | cat', realCmd) + # replace abuse retrieving (possible no-network): + realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+', + 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd) + # execute action: + return _actions.CommandAction.executeCmd(realCmd, timeout=timeout) + + def testComplexMailActionMultiLog(self): + testJailsActions = ( + # mail-whois-lines -- + ('j-mail-whois-lines', + 'mail-whois-lines[' + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s", ' + + # 2 logs to test grep from multiple logs: + 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' + '_whois_command="echo \'-- information about --\'"' + ']', + { + 'ip4-ban': ( + 'The IP 87.142.124.10 has just been banned by Fail2Ban after', + '100 attempts against j-mail-whois-lines.', + 'Here is more information about 87.142.124.10 :', + '-- information about 87.142.124.10 --', + 'Lines containing failures of 87.142.124.10 (max 2)', + 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + ), + }), + # complain -- + ('j-complain-abuse', + 'complain[' + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' + + # 2 logs to test grep from multiple logs: + 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' + ']', + { + 'ip4-ban': ( + 'Lines containing failures of 87.142.124.10 (max 2)', + 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + # both abuse mails should be separated with space: + 'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', + ), + }), + ) + server = TestServer() + transm = server._Server__transm + cmdHandler = transm._Transmitter__commandHandler + + for jail, act, tests in testJailsActions: + stream = self.getDefaultJailStream(jail, act) + + # for cmd in stream: + # print(cmd) + + # transmit jail to the server: + for cmd in stream: + # command to server: + ret, res = transm.proceed(cmd) + self.assertEqual(ret, 0) + + jails = server._Server__jails + + for jail, act, tests in testJailsActions: + # print(jail, jails[jail]) + for a in jails[jail].actions: + action = jails[jail].actions[a] + logSys.debug('# ' + ('=' * 50)) + logSys.debug('# == %-44s ==', jail + ' - ' + action._name) + logSys.debug('# ' + ('=' * 50)) + # wrap default command processor: + action.executeCmd = self._executeMailCmd + # test ban : + self.pruneLog('# === ban ===') + action.ban({'ip': IPAddr('87.142.124.10'), + 'failures': 100, + }) + self.assertLogged(*tests['ip4-ban'], all=True) From ec7bb0d6c9c06dcd9a09d8bd7338d9cfc4359a0d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 20:12:49 +0100 Subject: [PATCH 47/91] python 3x compatibility fix (positional arguments vs named arguments) --- fail2ban/client/configreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index caab67aa..a72ca1e9 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -141,7 +141,7 @@ class ConfigReader(): def get(self, sec, opt, raw=False, vars={}): if self._cfg is not None: - return self._cfg.get(sec, opt, raw, vars) + return self._cfg.get(sec, opt, raw=raw, vars=vars) return None def getOptions(self, *args, **kwargs): From 5678d08a79e8cfcba15389cd8e1bbfb10bb9ff68 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 26 Nov 2016 16:50:37 +0100 Subject: [PATCH 48/91] filter.d/dovecot.conf update: - fixes failregex, that ignores failures through some irrelevant info (closes #1623); - ignores whole additionally irrelevant info in anchored regex before fixed failure data `\((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\)` - review, IPv6 compatibility fix, non-capturing groups --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/dovecot.conf | 11 ++++++----- fail2ban/tests/files/logs/dovecot | 5 +++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca7864fc..ec8bdda3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,8 @@ releases. * Monit config: scripting is not supported in path (gh-1556) * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) +* `filter.d/dovecot.conf` + - Fixed failregex ignores failures through some not relevant info (gh-1623) * `filter.d/ignorecommands/apache-fakegooglebot` - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) diff --git a/THANKS b/THANKS index 64de43a4..7d9137d7 100644 --- a/THANKS +++ b/THANKS @@ -119,6 +119,7 @@ Thomas Mayer Tom Pike Tom Hendrikx Tomas Pihl +Thomas Skierlo (phaleas) Tony Lawrence Tomasz Ciolek Tyler diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 136a3947..a3dc60bb 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -9,11 +9,11 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) -failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=(, lip=(\d{1,3}\.){3}\d{1,3})?(, TLS( handshaking(: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ - ^%(__prefix_line)s(Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ - ^%(__prefix_line)s(auth|auth-worker\(\d+\)): (pam|passwd-file)\(\S+,\): unknown user\s*$ - ^%(__prefix_line)s(auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ +failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ + ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ + ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,\): unknown user\s*$ + ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ ignoreregex = @@ -30,3 +30,4 @@ journalmatch = _SYSTEMD_UNIT=dovecot.service # Author: Martin Waschbuesch # Daniel Black (rewrote with begin and end anchors) # Martin O'Neal (added LDAP authentication failure regex) +# Sergey G. Brester aka sebres (reviewed, optimized, IPv6-compatibility) diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 627b8dc8..987f89f1 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -73,3 +73,8 @@ Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempt # failJSON: { "time": "2005-03-23T06:10:52", "match": true , "host": "52.37.139.121" } Mar 23 06:10:52 auth: Info: ldap(dog,52.37.139.121,): invalid credentials + +# failJSON: { "time": "2005-07-26T11:11:21", "match": true , "host": "192.0.2.1" } +Jul 26 11:11:21 hostname dovecot: imap-login: Disconnected: Too many invalid commands (tried to use disallowed plaintext auth): user=, rip=192.0.2.1, lip=192.168.1.1, session= +# failJSON: { "time": "2005-07-26T11:12:19", "match": true , "host": "192.0.2.2" } +Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid commands (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.0.2.2, lip=192.168.1.1, TLS, session= From c06084d7d98ccd8a7ce28d18fc0f81402fc9752d Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 11:04:37 +0100 Subject: [PATCH 49/91] _start_params - fix: symlinks should be absolute paths --- fail2ban/tests/fail2banclienttestcase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index b64ecb38..e6cc46cb 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -138,8 +138,8 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"): """Filters list of 'files' to contain only directories (under dir)""" return [f for f in files if isdir(pjoin(dir, f))] shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) - os.symlink(pjoin(STOCK_CONF_DIR, "action.d"), pjoin(cfg, "action.d")) - os.symlink(pjoin(STOCK_CONF_DIR, "filter.d"), pjoin(cfg, "filter.d")) + os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "action.d")), pjoin(cfg, "action.d")) + os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "filter.d")), pjoin(cfg, "filter.d")) # replace fail2ban params (database with memory): r = re.compile(r'^dbfile\s*=') for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True): @@ -424,7 +424,7 @@ class Fail2banClientTest(Fail2banClientServerBase): self.execSuccess(startparams, "-vvd") self.assertLogged("Loading files") self.assertLogged("logtarget") - + @with_tmpdir @with_kill_srv def testClientStartBackgroundInside(self, tmp): From 931eab84b51f988b45e51844833c4a0807c2b7d4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 11:28:27 +0100 Subject: [PATCH 50/91] `filter.d/apache-modsecurity.conf` - fixed for newer version (one space, closes gh-1626) reviewed and optimized: - non-greedy catch-all replaced for safer match - unneeded catch-all anchoring removed - non-capturing groups --- config/filter.d/apache-modsecurity.conf | 3 ++- fail2ban/tests/files/logs/apache-modsecurity | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/filter.d/apache-modsecurity.conf b/config/filter.d/apache-modsecurity.conf index ad7e9b24..13e9c5ea 100644 --- a/config/filter.d/apache-modsecurity.conf +++ b/config/filter.d/apache-modsecurity.conf @@ -10,9 +10,10 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s ModSecurity: (\[.*?\] )*Access denied with code [45]\d\d.*$ +failregex = ^%(_apache_error_client)s ModSecurity:\s+(?:\[(?:\w+ \"[^\"]*\"|[^\]]*)\]\s*)*Access denied with code [45]\d\d ignoreregex = # https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats # Author: Daniel Black +# Sergey G. Brester aka sebres (review, optimization) \ No newline at end of file diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index d46d8ab4..3ca2e074 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -1,5 +1,5 @@ # failJSON: { "time": "2013-12-23T13:12:31", "match": true , "host": "173.255.225.101" } [Mon Dec 23 13:12:31 2013] [error] [client 173.255.225.101] ModSecurity: [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_crs_21_protocol_anomalies.conf"] [line "47"] [id "960015"] [rev "1"] [msg "Request Missing an Accept Header"] [severity "NOTICE"] [ver "OWASP_CRS/2.2.8"] [maturity "9"] [accuracy "9"] [tag "OWASP_CRS/PROTOCOL_VIOLATION/MISSING_HEADER_ACCEPT"] [tag "WASCTC/WASC-21"][tag "OWASP_TOP_10/A7"] [tag "PCI/6.5.10"] Access denied with code 403 (phase 2). Operator EQ matched 0 at REQUEST_HEADERS. [hostname "www.mysite.net"] [uri "/"] [unique_id "Urf@f12qgHIAACrFOlgAAABA"] -# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69" } -[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] +# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69", "desc": "additional entry (and exact one space)" } +[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] From b8c41dcb491b6ed4ceb2e89127b6e1185e4ca344 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 11:31:51 +0100 Subject: [PATCH 51/91] ChangeLog update --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index ec8bdda3..29124261 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,9 @@ releases. * Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); * FIPS compliant, use sha1 instead of md5 if it not allowed (see gh-1540) * Monit config: scripting is not supported in path (gh-1556) +* `filter.d/apache-modsecurity.conf` + - Fixed for newer version (one space, gh-1626), optimized: non-greedy catch-all + replaced for safer match, unneeded catch-all anchoring removed, non-capturing * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) * `filter.d/dovecot.conf` From 5d5ab274350035cd9899ab15652f97216dd5d759 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 13:17:36 +0100 Subject: [PATCH 52/91] small amend: removed unreachable code + coverage increase --- fail2ban/server/datetemplate.py | 2 -- fail2ban/tests/datedetectortestcase.py | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index f670cbbd..1d0b014b 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -219,7 +219,6 @@ class DateEpoch(DateTemplate): if dateMatch: # extract part of format which represents seconds since epoch return (float(dateMatch.group(1)), dateMatch) - return None class DatePatternRegex(DateTemplate): @@ -338,4 +337,3 @@ class DateTai64n(DateTemplate): seconds_since_epoch = value[2:17] # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) - return None diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 695f099c..15cb6f2d 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -132,14 +132,15 @@ class DateDetectorTest(LogCaptureTestCase): (True, True, "1106513999.000", None), # Regular epoch with millisec (True, True, "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match) (False, True, "audit(1106513999.000:987)", "1106513999.000"), # SELinux + (True, True, "no date line", None), # no date in string ): - logSys.debug('== test %r', (anchored, bound, sdate)) + if rdate is None and sdate != "no date line": rdate = sdate + logSys.debug('== test %r', (anchored, bound, sdate, rdate)) for should_match, prefix in ( - (True, ""), + (rdate is not None, ""), (not anchored, "bogus-prefix "), (False, "word-boundary") ): - if rdate is None: rdate = sdate log = prefix + sdate + "[sshd] error: PAM: Authentication failure" # if not allowed boundary test: if not bound and prefix == "word-boundary": continue @@ -283,6 +284,16 @@ class DateDetectorTest(LogCaptureTestCase): finally: datedetector.logLevel = self.__old_eff_level + def testWrongTemplate(self): + t = DatePatternRegex('(%ExY%Exm%Exd') + # lazy compiling used, so try match: + self.assertRaises(Exception, t.matchDate, '(20050101') + self.assertLogged("Compile %r failed" % t.name) + # abstract: + t = DateTemplate() + self.assertRaises(Exception, t.getDate, 'no date line') + + iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): From 39c343bd0699ae98d49a98aeeefaa4a9ecc6347a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 15:18:31 +0100 Subject: [PATCH 53/91] better reorder templates handling, code coverage increase (a small part of _reorderTemplate was not covered at all) --- fail2ban/server/datedetector.py | 52 +++++++++++++------------- fail2ban/tests/datedetectortestcase.py | 4 +- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 01cb43fe..bef3be6e 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -92,7 +92,7 @@ class DateDetectorCache(object): if self.__templates: return self.__templates with self.__lock: - if self.__templates: + if self.__templates: # pragma: no cover - race-condition + multi-threaded environment only return self.__templates self._addDefaultTemplate() return self.__templates @@ -324,7 +324,7 @@ class DateDetector(object): ddtempl = self.__templates[i] template = ddtempl.template if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): - if logSys.getEffectiveLevel() <= logLevel-1: + if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) ignoreBySearch = i @@ -452,7 +452,7 @@ class DateDetector(object): try: date = template.getDate(line, timeMatch[0]) if date is not None: - if logSys.getEffectiveLevel() <= logLevel: + if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug logSys.log(logLevel, " got time %f for %r using template %s", date[0], date[1].group(1), template.name) return date @@ -478,29 +478,31 @@ class DateDetector(object): weight = ddtempl.weight ## try to move faster (first if unused available, or half of part to current template position): pos = self.__firstUnused if self.__firstUnused < num else num // 2 - pweight = templates[pos].weight ## don't move too often (multiline logs resp. log's with different date patterns), ## if template not used too long, replace it also : - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r", - num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) - if not pweight or weight > pweight or templates[pos].lastUsed < untime: - ## if not larger (and target position recently used) - move slow (exact 1 position): - if weight <= pweight and templates[pos].lastUsed > untime: - pos = num-1 - ## if still smaller and template at position used, don't move: - pweight = templates[pos].weight - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r", - num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) - if weight < pweight and templates[pos].lastUsed > untime: - return - del templates[num] - templates[pos:0] = [ddtempl] - ## correct first unused: - while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits: - self.__firstUnused += 1 + def _moveable(): + pweight = templates[pos].weight if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " -> moved template #%02i -> #%02i", num, pos) - return pos + logSys.log(logLevel, " -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r", + num, pos, weight, pweight, ddtempl.hits, templates[pos].hits) + return weight > pweight or untime > templates[pos].lastUsed + ## + ## if not moveable (smaller weight or target position recently used): + if not _moveable(): + ## try to move slow (exact 1 position): + if pos == num-1: + return num + pos = num-1 + ## if still smaller and template at position used, don't move: + if not _moveable(): + return num + ## move: + del templates[num] + templates[pos:0] = [ddtempl] + ## correct first unused: + while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits: + self.__firstUnused += 1 + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " -> moved template #%02i -> #%02i", num, pos) + return pos return num diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 15cb6f2d..5b32a7e9 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -250,12 +250,12 @@ class DateDetectorTest(LogCaptureTestCase): ("030324 0:04:00", "server mysqld[1000]: 030324 0:04:00 [Warning] Access denied ..." " foreign-input just some free text 2003-03-07 17:05:01 test", 10), # distance collision detection (first date should be found): - ("Sep 16 21:30:26", "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 10), + ("Sep 16 21:30:26", "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 15), # just to test sorting: ("2005-10-07 06:09:42", "server mysqld[5906]: 2005-10-07 06:09:42 5907 [Warning] Access denied", 20), ("2005-10-08T15:26:18.237955", "server mysqld[5906]: 2005-10-08T15:26:18.237955 6 [Note] Access denied", 20), # date format changed again: - ("051009 10:05:30", "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 20), + ("051009 10:05:30", "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 50), ): logSys.debug('== test: %r', (debit, line, cnt)) for i in range(cnt): From 8018796b45b91e5cc8894978077b45234e299e6b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 17:17:48 +0100 Subject: [PATCH 54/91] wrong indentation (important code-piece in if log-level only) --- fail2ban/server/datedetector.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index bef3be6e..cccbf71a 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -326,8 +326,8 @@ class DateDetector(object): if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) - match = template.matchDate(line) - ignoreBySearch = i + match = template.matchDate(line) + ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] if logSys.getEffectiveLevel() <= logLevel-1: @@ -357,7 +357,6 @@ class DateDetector(object): logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: - self.__lastTemplIdx = 0x7fffffff logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) found = None, 0x7fffffff, 0x7fffffff, -1 i = 0 @@ -414,7 +413,7 @@ class DateDetector(object): self.__lastPos = distance, line[distance-1:distance] self.__lastEndPos = endpos, line[endpos:endpos+1] # if not first - try to reorder current template (bubble up), they will be not sorted anymore: - if i: + if i and i != self.__lastTemplIdx: i = self._reorderTemplate(i) self.__lastTemplIdx = i # return tuple with match and template reference used for parsing: From 425170cef3d9c2f3ac98361d931111575a8a8fc2 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 18:37:12 +0100 Subject: [PATCH 55/91] code review, makes the test cases workable, added dev-notes --- config/filter.d/mongodb-auth.conf | 26 ++++++++++++---- fail2ban/tests/files/logs/mongodb-auth | 42 +++++++++++++++----------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf index 43462e73..66c27abb 100644 --- a/config/filter.d/mongodb-auth.conf +++ b/config/filter.d/mongodb-auth.conf @@ -9,17 +9,31 @@ # auth = true # -[Init] -maxlines = 10 - [Definition] -failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]*\s+\[conn(?P=__connid)\] end connection +#failregex = ^\s+\[initandlisten\] connection accepted from :\d+ \#(?P<__connid>\d+) \(1 connection now open\)\s+\[conn(?P=__connid)\] Failed to authenticate\s+ +failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]+\s+\[conn(?P=__connid)\] end connection ignoreregex = +[Init] +maxlines = 10 + # DEV Notes: # +# Regarding the multiline regex: +# +# There can be a nunber of non-related lines between the first and second part +# of this regex maxlines of 10 is quite generious. +# +# Note the capture __connid, includes the connection ID, used in second part of regex. +# +# The first regex is commented out (but will match also), because it is better to use +# the host from "end connection" line (uncommented above): +# - it has the same prefix, searching begins directly with failure message +# (so faster, because ignores success connections at all) +# - it is not so vulnerable in case of possible race condition +# # Log example: # 2016-10-20T09:54:27.108+0200 [initandlisten] connection accepted from 127.0.0.1:53276 #1 (1 connection now open) # 2016-10-20T09:54:27.109+0200 [conn1] authenticate db: test { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } @@ -30,6 +44,6 @@ ignoreregex = # 2016-11-09T11:55:58.892+0100 [conn1510] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch # 2016-11-09T11:55:58.894+0100 [conn1510] end connection 127.0.0.1:54266 (0 connections now open) # -# Authors: Alexander Finkhäuser and sebres -# +# Authors: Alexander Finkhäuser +# Sergey G. Brester (sebres) diff --git a/fail2ban/tests/files/logs/mongodb-auth b/fail2ban/tests/files/logs/mongodb-auth index a9c7b99b..8a308892 100644 --- a/fail2ban/tests/files/logs/mongodb-auth +++ b/fail2ban/tests/files/logs/mongodb-auth @@ -1,24 +1,30 @@ -# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.168.1.35" } -2016-11-20T00:04:00.110+0200 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin -2016-11-20T00:04:00.111+0200 [conn1] end connection 192.168.1.35:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:04:00.110+0100 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.0.2.35" } +2016-11-20T00:04:00.111+0100 [conn1] end connection 192.0.2.35:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.171" } -2016-11-20T00:24:00.110+0200 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin -2016-11-20T00:24:00.111+0200 [conn5] end connection 220.95.238.171:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.171" } +2016-11-20T00:24:00.111+0100 [conn5] end connection 192.0.2.171:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.176" } -2016-11-20T00:24:00.110+0200 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch -2016-11-20T00:24:00.111+0200 [conn334] end connection 220.95.238.176:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.176" } +2016-11-20T00:24:00.111+0100 [conn334] end connection 192.0.2.176:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "167.96.268.1" } -2016-11-20T00:24:00.110+0200 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch -2016-11-20T00:24:00.111+0200 [conn56] end connection 167.96.268.1:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.1" } +2016-11-20T00:24:00.111+0100 [conn56] end connection 192.0.2.1:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": false , "host": "127.0.0.1" } -2016-11-10T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) -2016-11-10T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) +# failJSON: { "match": false } +2016-11-20T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) -# failJSON: { "time": "2016-11-10T13:07:49", "match": false , "host": "177.13.20.178" } -2016-11-10T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } -2016-11-10T13:07:49.834+0100 [conn2271] end connection 177.13.20.178:60268 (3 connections now open) +# failJSON: { "match": false } +2016-11-20T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# failJSON: { "time": "2016-11-20T13:07:49", "match": false , "host": "192.0.2.178" } +2016-11-20T13:07:49.834+0100 [conn2271] end connection 192.0.2.178:60268 (3 connections now open) From 67c14afd8ee1c1cd3edc417474ab7afc65e1b393 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 18:51:23 +0100 Subject: [PATCH 56/91] ChangeLog entry added + jail.conf review --- ChangeLog | 4 ++++ config/jail.conf | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2a6b5733..45d3ad99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,10 @@ releases. * New Actions: - `action.d/npf.conf` for NPF, the latest packet filter for NetBSD +* New Filters: + - `filter.d/mongodb-auth.conf` for MongoDB (document-oriented NoSQL database engine) + + ### Enhancements * DateTemplate regexp extended with the word-end boundary, additionally to word-start boundary diff --git a/config/jail.conf b/config/jail.conf index 0778a318..4d9f3345 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,8 +731,9 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s -# Log wrong MongoDB auth (for details how ... see filter ...) +# Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] +# change port when running with "--shardsvr" or "--configsvr" runtime operation port = 27017 logpath = /var/log/mongodb/mongodb.log From feae7370ced6eabb8a542f4c7d7c3bba53c93aa7 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 28 Nov 2016 23:19:24 +0100 Subject: [PATCH 57/91] Update THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index c8a019a6..8f746f29 100644 --- a/THANKS +++ b/THANKS @@ -110,6 +110,7 @@ SATO Kentaro Sean DuBois Sebastian Arcus Serg G. Brester +Sergey Safarov Sireyessire silviogarbes Stefan Tatschner From 556a9373ce42c934825c090370e1abe7e1670eb5 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 28 Nov 2016 23:40:33 +0100 Subject: [PATCH 58/91] Update ChangeLog --- ChangeLog | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index dc37e55c..e86fce18 100644 --- a/ChangeLog +++ b/ChangeLog @@ -49,14 +49,12 @@ releases. - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) - optional port part after host (see gh-1533, gh-1581) - ### New Features * New Actions: - `action.d/npf.conf` for NPF, the latest packet filter for NetBSD - * New Filters: - `filter.d/mongodb-auth.conf` for MongoDB (document-oriented NoSQL database engine) - + (gh-1586, gh-1606 and gh-1607) ### Enhancements * DateTemplate regexp extended with the word-end boundary, additionally to From e550850b9c3f04461624bf91ae438c920ac4d215 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:34:44 -0500 Subject: [PATCH 59/91] BF: added missing entires into MANIFEST --- MANIFEST | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/MANIFEST b/MANIFEST index e0d7398c..b12e3163 100644 --- a/MANIFEST +++ b/MANIFEST @@ -33,12 +33,14 @@ config/action.d/iptables-new.conf config/action.d/iptables-xt_recent-echo.conf config/action.d/mail-buffered.conf config/action.d/mail.conf +config/action.d/mail-whois-common.conf config/action.d/mail-whois.conf config/action.d/mail-whois-lines.conf config/action.d/mynetwatchman.conf config/action.d/nftables-allports.conf config/action.d/nftables-common.conf config/action.d/nftables-multiport.conf +config/action.d/npf.conf config/action.d/nsupdate.conf config/action.d/osx-afctl.conf config/action.d/osx-ipfw.conf @@ -54,6 +56,7 @@ config/action.d/sendmail-whois-ipmatches.conf config/action.d/sendmail-whois-lines.conf config/action.d/sendmail-whois-matches.conf config/action.d/shorewall.conf +config/action.d/shorewall-ipset-proto6.conf config/action.d/smtp.py config/action.d/symbiosis-blacklist-allports.conf config/action.d/ufw.conf @@ -69,6 +72,7 @@ config/filter.d/apache-modsecurity.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf +config/filter.d/apache-pass.conf config/filter.d/apache-shellshock.conf config/filter.d/assp.conf config/filter.d/asterisk.conf @@ -81,11 +85,13 @@ config/filter.d/cyrus-imap.conf config/filter.d/directadmin.conf config/filter.d/dovecot.conf config/filter.d/dropbear.conf +config/filter.d/drupal-auth.conf config/filter.d/ejabberd-auth.conf config/filter.d/exim-common.conf config/filter.d/exim.conf config/filter.d/exim-spam.conf config/filter.d/freeswitch.conf +config/filter.d/froxlor-auth.conf config/filter.d/groupoffice.conf config/filter.d/gssftpd.conf config/filter.d/guacamole.conf @@ -95,6 +101,7 @@ config/filter.d/ignorecommands config/filter.d/ignorecommands/apache-fakegooglebot config/filter.d/kerio.conf config/filter.d/lighttpd-auth.conf +config/filter.d/mongodb-auth.conf config/filter.d/monit.conf config/filter.d/murmur.conf config/filter.d/mysqld-auth.conf @@ -150,6 +157,7 @@ config/paths-opensuse.conf config/paths-osx.conf CONTRIBUTING.md COPYING +.coveragerc DEVELOP doc/run-rootless.txt fail2ban-2to3 @@ -206,6 +214,7 @@ fail2ban/tests/banmanagertestcase.py fail2ban/tests/clientreadertestcase.py fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/common.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local @@ -257,6 +266,7 @@ fail2ban/tests/files/logs/apache-modsecurity fail2ban/tests/files/logs/apache-nohome fail2ban/tests/files/logs/apache-noscript fail2ban/tests/files/logs/apache-overflows +fail2ban/tests/files/logs/apache-pass fail2ban/tests/files/logs/apache-shellshock fail2ban/tests/files/logs/assp fail2ban/tests/files/logs/asterisk @@ -270,10 +280,12 @@ fail2ban/tests/files/logs/cyrus-imap fail2ban/tests/files/logs/directadmin fail2ban/tests/files/logs/dovecot fail2ban/tests/files/logs/dropbear +fail2ban/tests/files/logs/drupal-auth fail2ban/tests/files/logs/ejabberd-auth fail2ban/tests/files/logs/exim fail2ban/tests/files/logs/exim-spam fail2ban/tests/files/logs/freeswitch +fail2ban/tests/files/logs/froxlor-auth fail2ban/tests/files/logs/groupoffice fail2ban/tests/files/logs/gssftpd fail2ban/tests/files/logs/guacamole @@ -281,6 +293,7 @@ fail2ban/tests/files/logs/haproxy-http-auth fail2ban/tests/files/logs/horde fail2ban/tests/files/logs/kerio fail2ban/tests/files/logs/lighttpd-auth +fail2ban/tests/files/logs/mongodb-auth fail2ban/tests/files/logs/monit fail2ban/tests/files/logs/murmur fail2ban/tests/files/logs/mysqld-auth @@ -357,6 +370,8 @@ files/gentoo-confd files/gentoo-initd files/ipmasq-ZZZzzz_fail2ban.rul files/logwatch/fail2ban +files/logwatch/fail2ban-0.8.log +files/logwatch/fail2ban-0.9.log files/macosx-initd files/monit/fail2ban files/nagios/check_fail2ban @@ -374,8 +389,11 @@ man/fail2ban-regex.1 man/fail2ban-regex.h2m man/fail2ban-server.1 man/fail2ban-server.h2m +man/fail2ban-testcases.1 +man/fail2ban-testcases.h2m man/generate-man man/jail.conf.5 +.pylintrc README.md README.Solaris RELEASE From 482252dbd4d6576a376158c5b49da82e5af8b7c4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:35:03 -0500 Subject: [PATCH 60/91] ENH: prep for 0.9.6 release (as of tomorrow) --- ChangeLog | 12 ++++++------ README.md | 6 +++--- RELEASE | 8 ++++---- fail2ban/version.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index e86fce18..0852a360 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,7 +6,7 @@ Fail2Ban: Changelog =================== -ver. 0.9.6 (2016/XX/XX) - wanna-be-released +ver. 0.9.6 (2016/12/10) - stretch-is-coming ----------- 0.9.x line is no longer heavily developed. If you are interested in @@ -18,7 +18,7 @@ releases. induced a subsequent error: last position of log file will be never retrieved (gh-795) * Fixed a distribution related bug within testReadStockJailConfForceEnabled (e.g. test-cases faults on Fedora, see gh-1353) -* Fixed pythonic filters and test scripts (running via wrong python version, +* Fixed pythonic filters and test scripts (running via wrong python version, uses "fail2ban-python" now); * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); @@ -33,7 +33,7 @@ releases. * `filter.d/dovecot.conf` - Fixed failregex ignores failures through some not relevant info (gh-1623) * `filter.d/ignorecommands/apache-fakegooglebot` - - Fixed error within apache-fakegooglebot, that will be called + - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) * `filter.d/assp.conf` - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) @@ -57,11 +57,11 @@ releases. (gh-1586, gh-1606 and gh-1607) ### Enhancements -* DateTemplate regexp extended with the word-end boundary, additionally to +* DateTemplate regexp extended with the word-end boundary, additionally to word-start boundary -* Introduces new command "fail2ban-python", as automatically created symlink to +* Introduces new command "fail2ban-python", as automatically created symlink to python executable, where fail2ban currently installed (resp. its modules are located): - - allows to use the same version, fail2ban currently running, e.g. in + - allows to use the same version, fail2ban currently running, e.g. in external scripts just via replace python with fail2ban-python: ```diff -#!/usr/bin/env python diff --git a/README.md b/README.md index c2ef6c70..998e43e2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.5 2016/07/15 + v0.9.6 2016/12/10 ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -39,8 +39,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.5.tar.bz2 - cd fail2ban-0.9.5 + tar xvfj fail2ban-0.9.6.tar.bz2 + cd fail2ban-0.9.6 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index 6ad9b52d..b879fa20 100644 --- a/RELEASE +++ b/RELEASE @@ -53,7 +53,7 @@ Preparation or an alternative for comparison with previous release - git diff 0.9.5 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST + git diff 0.9.6 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST sort MANIFEST | uniq | sponge MANIFEST * Run:: @@ -70,7 +70,7 @@ Preparation * clean up current directory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.5/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.6/ * Only differences should be files that you don't want distributed. @@ -83,7 +83,7 @@ Preparation * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.5.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.6.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -106,7 +106,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.5 + git tag -s 0.9.6 Pre Release =========== diff --git a/fail2ban/version.py b/fail2ban/version.py index 57aa3c28..194918d7 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.5" +version = "0.9.6" From 3605155978efc95a2a44337645994dafc2f2b366 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:36:08 -0500 Subject: [PATCH 61/91] updated man pages --- man/fail2ban-client.1 | 6 +++--- man/fail2ban-regex.1 | 4 ++-- man/fail2ban-server.1 | 6 +++--- man/fail2ban-testcases.1 | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index f67e71ea..ff3e5f77 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-CLIENT "1" "July 2016" "fail2ban-client v0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-CLIENT "1" "December 2016" "fail2ban-client v0.9.6" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.9.5 reads log file that contains password failure report +Fail2Ban v0.9.6 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 1db372cc..1ed2c327 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-REGEX "1" "July 2016" "fail2ban-regex 0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-REGEX "1" "December 2016" "fail2ban-regex 0.9.6" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 96f0c3e5..5278302c 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-SERVER "1" "July 2016" "fail2ban-server v0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-SERVER "1" "December 2016" "fail2ban-server v0.9.6" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.9.5 reads log file that contains password failure report +Fail2Ban v0.9.6 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 1c2f1a8e..658555ac 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-TESTCASES "1" "July 2016" "fail2ban-testcases 0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-TESTCASES "1" "December 2016" "fail2ban-testcases 0.9.6" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 4a1fd888f030a6a329e2a3fdf51130b8ee76d570 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 11 Dec 2016 00:49:09 -0500 Subject: [PATCH 62/91] Carry on development --- ChangeLog | 17 ++++++++++++++++- README.md | 2 +- RELEASE | 2 +- fail2ban/version.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0852a360..b9fadce5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,13 +6,28 @@ Fail2Ban: Changelog =================== -ver. 0.9.6 (2016/12/10) - stretch-is-coming + +ver. 0.9.x (2016/??/??) - wanna-be-released ----------- 0.9.x line is no longer heavily developed. If you are interested in new features (e.g. IPv6 support), please consider 0.10 branch and its releases. +### Fixes + +### New Features +* New Actions: + +* New Filters: + + +### Enhancements + + +ver. 0.9.6 (2016/12/10) - stretch-is-coming +----------- + ### Fixes * Misleading add resp. enable of (already available) jail in database, that induced a subsequent error: last position of log file will be never retrieved (gh-795) diff --git a/README.md b/README.md index 998e43e2..3f0e3253 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.6 2016/12/10 + v0.9.6.dev0 2016/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/RELEASE b/RELEASE index b879fa20..c4f62d7a 100644 --- a/RELEASE +++ b/RELEASE @@ -190,7 +190,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.7 (2016/XX/XXX) - wanna-be-released + ver. 0.9.8 (2016/XX/XXX) - wanna-be-released ----------- ### Fixes diff --git a/fail2ban/version.py b/fail2ban/version.py index 194918d7..c2a8dc59 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.6" +version = "0.9.6.dev0" From 31a1560eaae663cb3b740829e3262198a909d1c4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 11 Dec 2016 15:13:11 -0500 Subject: [PATCH 63/91] minor typos (thanks Vincent Lefevre, Debian #847785) --- config/action.d/firewallcmd-rich-logging.conf | 2 +- config/action.d/firewallcmd-rich-rules.conf | 2 +- config/filter.d/assp.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index 1b88c2d9..c4a8b6f7 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -29,7 +29,7 @@ actioncheck = # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' port port='' protocol='' log prefix='f2b-' level='' limit value='/m' " # service name example: # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' service name='' log prefix='f2b-' level='' limit value='/m' " -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp +# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index 4e39df54..5bf10b03 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -27,7 +27,7 @@ actioncheck = # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' port port='' protocol='' " # service name example: # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' service name='' " -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp +# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='' port port='$p' protocol='' "; done diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 278e25cb..ddf18f33 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -8,7 +8,7 @@ # [Definition] -# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibilty reasons. +# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibility reasons. __assp_actions = (?:dropping|refusing) From fb27d9ce83191ecf2887b84453f7ea0dae9e4c8f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 10:59:53 +0100 Subject: [PATCH 64/91] fail2ban-regex: fix for systemd-journal (see gh-1657) --- fail2ban/client/fail2banregex.py | 31 ++++++++++++++----------------- fail2ban/server/filtersystemd.py | 8 ++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index ff55c417..d111e09c 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -41,12 +41,12 @@ from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError try: # pragma: no cover - from systemd import journal from ..server.filtersystemd import FilterSystemd except ImportError: - journal = None + FilterSystemd = None from ..version import version +from .jailreader import JailReader from .filterreader import FilterReader from ..server.filter import Filter, FileContainer from ..server.failregex import RegexException @@ -80,7 +80,7 @@ def pprint_list(l, header=None): s = '' output( s + "| " + "\n| ".join(l) + '\n`-' ) -def journal_lines_gen(myjournal): # pragma: no cover +def journal_lines_gen(flt, myjournal): # pragma: no cover while True: try: entry = myjournal.get_next() @@ -88,7 +88,7 @@ def journal_lines_gen(myjournal): # pragma: no cover continue if not entry: break - yield FilterSystemd.formatJournalEntry(entry) + yield flt.formatJournalEntry(entry) def get_opt_parser(): # use module docstring for help output @@ -524,25 +524,22 @@ class Fail2banRegex(object): except IOError as e: output( e ) return False - elif cmd_log == "systemd-journal": # pragma: no cover - if not journal: + elif cmd_log.startswith("systemd-journal"): # pragma: no cover + if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + output( "Use systemd journal" ) + output( "Use encoding : %s" % self._encoding ) + backend, beArgs = JailReader.extractOptions(cmd_log) + flt = FilterSystemd(None, **beArgs) + flt.setLogEncoding(self._encoding) + myjournal = flt.getJournalReader() journalmatch = self._journalmatch self.setDatePattern(None) if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) - return False + flt.addJournalMatch(journalmatch) output( "Use journal match : %s" % " ".join(journalmatch) ) - test_lines = journal_lines_gen(myjournal) + test_lines = journal_lines_gen(flt, myjournal) else: output( "Use single line : %s" % shortstr(cmd_log) ) test_lines = [ cmd_log ] diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 0d720a5e..d858142b 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -178,6 +178,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalMatch(self): return self.__matches + ## + # Get journal reader + # + # @return journal reader + + def getJournalReader(self): + return self.__journal + ## # Format journal log entry into syslog style # From 2009f1c4346597dcc4fd27151d220ea4a7806fdd Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 10:59:53 +0100 Subject: [PATCH 65/91] fail2ban-regex: fix for systemd-journal (see gh-1657) --- fail2ban/client/fail2banregex.py | 31 ++++++++++++++----------------- fail2ban/server/filtersystemd.py | 8 ++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 71f50955..13fa35d9 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -43,12 +43,12 @@ from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError try: - from systemd import journal from ..server.filtersystemd import FilterSystemd except ImportError: - journal = None + FilterSystemd = None from ..version import version +from .jailreader import JailReader from .filterreader import FilterReader from ..server.filter import Filter, FileContainer from ..server.failregex import RegexException @@ -82,7 +82,7 @@ def pprint_list(l, header=None): s = '' output( s + "| " + "\n| ".join(l) + '\n`-' ) -def journal_lines_gen(myjournal): +def journal_lines_gen(flt, myjournal): # pragma: no cover while True: try: entry = myjournal.get_next() @@ -90,7 +90,7 @@ def journal_lines_gen(myjournal): continue if not entry: break - yield FilterSystemd.formatJournalEntry(entry) + yield flt.formatJournalEntry(entry) def get_opt_parser(): # use module docstring for help output @@ -513,25 +513,22 @@ class Fail2banRegex(object): except IOError as e: output( e ) return False - elif cmd_log == "systemd-journal": # pragma: no cover - if not journal: + elif cmd_log.startswith("systemd-journal"): # pragma: no cover + if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + output( "Use systemd journal" ) + output( "Use encoding : %s" % self.encoding ) + backend, beArgs = JailReader.extractOptions(cmd_log) + flt = FilterSystemd(None, **beArgs) + flt.setLogEncoding(self.encoding) + myjournal = flt.getJournalReader() journalmatch = self._journalmatch self.setDatePattern(None) if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) - return False + flt.addJournalMatch(journalmatch) output( "Use journal match : %s" % " ".join(journalmatch) ) - test_lines = journal_lines_gen(myjournal) + test_lines = journal_lines_gen(flt, myjournal) else: output( "Use single line : %s" % shortstr(cmd_log) ) test_lines = [ cmd_log ] diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 3023155c..908112a7 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -174,6 +174,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover v = Filter.uni_decode(x, self.getLogEncoding()) return v + ## + # Get journal reader + # + # @return journal reader + + def getJournalReader(self): + return self.__journal + ## # Format journal log entry into syslog style # From f8d35a7c9c8b783b6cc40ef13c0462a5ad8dcaf0 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 11:16:17 +0100 Subject: [PATCH 66/91] changelog entry --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b9fadce5..0fd91b70 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ new features (e.g. IPv6 support), please consider 0.10 branch and its releases. ### Fixes +* Fixed a systemd-journal handling in fail2ban-regex (gh-1657) ### New Features * New Actions: From a9523aefbb9b179e6a18ae163d44dc9f9b4b7695 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 12:51:11 +0100 Subject: [PATCH 67/91] sshd.conf: fixed non-anchored part of regex (misleading match of colon inside IPv6 address instead of `: ` in the reason-part by missing space). --- ChangeLog | 4 ++++ config/filter.d/sshd.conf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0fd91b70..126bd2e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,10 @@ releases. ### Fixes * Fixed a systemd-journal handling in fail2ban-regex (gh-1657) +* filter.d/sshd.conf + - Fixed non-anchored part of failregex (misleading match of colon inside + IPv6 address instead of `: ` in the reason-part by missing space, gh-1658) + (0.10th resp. IPv6 relevant only, amend for gh-1479) ### New Features * New Actions: diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 35cd8754..d5a66cc8 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) + ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .*? from (?: port \d+)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ From bf872213bdad3228510a977740eaac3aac2d612a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 13:48:17 +0100 Subject: [PATCH 68/91] amend for 7019640eb3d1f901e52b4c00cb6d029e82fdda17 (fix-gh-1658): sshd test-cases extended with IPv6 to cover this fix --- fail2ban/tests/files/logs/sshd | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 15ec4274..0879a2da 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -9,8 +9,14 @@ May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-i #2 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 50273 ssh2 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 12345 +# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:11 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 #3 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } @@ -118,6 +124,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +# failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } +Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -128,6 +136,8 @@ Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI +# failJSON: { "time": "2005-07-05T18:22:45", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Failed publickey for ..." } +Jul 05 18:22:45 mercury sshd[4670]: Failed publickey for graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI # failJSON: { "match": false } Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth] From 6f190b6e611cfee8796f6720e8df17c73e0e63ee Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 11 Jan 2017 19:04:33 +0100 Subject: [PATCH 69/91] readme.md: added IPv6 launch logo for 0.10th branch Closes gh-1647 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index be0be636..ee654acb 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. + + | Since v0.10 fail2ban supports the matching of the IPv6 addresses. +------|------ This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs are available in fail2ban(1) manpage and on the website http://www.fail2ban.org From 4a65e069e1437d165b3265d6caba623e02327462 Mon Sep 17 00:00:00 2001 From: oliverdorn Date: Thu, 12 Jan 2017 22:07:46 +0100 Subject: [PATCH 70/91] Solution for issue #1665 Solves the issue of authentic GoogleBots being banned by apache-fakegooglebots. --- files/debian-initd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/debian-initd b/files/debian-initd index 0d2bed9a..d6660215 100755 --- a/files/debian-initd +++ b/files/debian-initd @@ -22,7 +22,7 @@ # rename this file: (sudo) mv /etc/init.d/fail2ban.init /etc/init.d/fail2ban # same with the logrotate file: (sudo) mv /etc/logrotate.d/fail2ban.logrotate /etc/logrotate.d/fail2ban # -PATH=/usr/sbin:/usr/bin:/sbin:/bin +PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin DESC="authentication failure monitor" NAME=fail2ban From ee3c787cc68f6d043c8702c8f53a566277e2ee02 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Jan 2017 19:06:17 +0100 Subject: [PATCH 71/91] Recognize restored (from database) tickets after restart (tell action restored state of the ticket); Prevent executing of several actions (e.g. mail, send-mail etc) on restart (bans were already notified). Test cases extended (smtp and by restart in ServerReloadTest). Closes gh-1141 Closes gh-921 --- config/action.d/complain.conf | 3 +- config/action.d/dshield.conf | 10 ++++- config/action.d/helpers-common.conf | 3 ++ config/action.d/mail-buffered.conf | 8 +++- config/action.d/mail-whois-lines.conf | 4 +- config/action.d/mail-whois.conf | 4 +- config/action.d/mail.conf | 7 +++- config/action.d/sendmail-buffered.conf | 4 +- config/action.d/sendmail-geoip-lines.conf | 3 +- .../sendmail-whois-ipjailmatches.conf | 4 +- config/action.d/sendmail-whois-ipmatches.conf | 4 +- config/action.d/sendmail-whois-lines.conf | 3 +- config/action.d/sendmail-whois-matches.conf | 4 +- config/action.d/sendmail-whois.conf | 4 +- config/action.d/sendmail.conf | 4 +- config/action.d/smtp.py | 2 + config/action.d/xarf-login-attack.conf | 7 +++- fail2ban/server/action.py | 4 +- fail2ban/server/actions.py | 4 ++ fail2ban/tests/action_d/test_smtp.py | 15 ++++++- fail2ban/tests/fail2banclienttestcase.py | 42 +++++++++++++++---- 21 files changed, 114 insertions(+), 29 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index e4ceb35f..d0156a44 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -58,7 +58,8 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = oifs=${IFS}; +actionban = %(_bypass_if_restored)s + oifs=${IFS}; IFS=.; SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index a0041986..35eaa3be 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -26,6 +26,10 @@ # configure how often the buffer is flushed). # +[INCLUDES] + +before = helpers-common.conf + [Definition] # Option: actionstart @@ -64,7 +68,8 @@ actioncheck = # few seconds out, are incorrect. See # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047 # -actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'` +actionban = %(_bypass_if_restored)s + TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'` DATETIME="`perl -e '@t=localtime(