diff --git a/ChangeLog b/ChangeLog index 8083eaff..7cb59dcf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -68,6 +68,12 @@ ver. 0.11.2-dev (20??/??/??) - development edition * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) ### New Features and Enhancements +* fail2ban-regex: + - speedup formatted output (bypass unneeded stats creation) + - extended with prefregex statistic + - more informative output for `datepattern` (e. g. set from filter) - pattern : description +* parsing of action in jail-configs considers space between action-names as separator also + (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 14995fed..84fcbdda 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -8,11 +8,14 @@ before = common.conf [Definition] _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) +# "\w{14,20}" will give support for IDs from 14 up to 20 characters long __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) -# "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:|)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ +prefregex = ^%(__prefix_line)s.+$ +failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index ca171915..e8b766c5 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,19 +21,20 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=(?:IPv6:|), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^rejecting commands from (\S* )?\[(?:IPv6:|)\] due to pre-greeting traffic after \d+ seconds$ - ^(?:\S+ )?\[(?:IPv6:|)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ - ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:|)\]$ + ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$ mdre-normal = -mdre-extra = ^(?:\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$ +mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection mdre-aggressive = %(mdre-extra)s diff --git a/config/jail.conf b/config/jail.conf index b30b669d..b2fd263b 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -209,16 +209,16 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. action_mw = %(action_)s - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(action_)s - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # @@ -230,7 +230,7 @@ action_xarf = %(action_)s # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 98fd9799..d0fe55dc 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -252,6 +252,8 @@ class Fail2banRegex(object): self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -292,8 +294,8 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - self.output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: @@ -453,19 +455,33 @@ class Fail2banRegex(object): lines = [] ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False + return None, 0, None if self._filter.getMaxLines() > 1: for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -651,7 +667,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index dc7852ae..f381576e 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -373,7 +373,7 @@ OPTION_EXTRACT_CRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( - r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL) + r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) def extractOptions(option): match = OPTION_CRE.match(option) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 1a3ee052..31b729b0 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -277,7 +277,8 @@ class Transmitter: value = command[2] self.__server.setPrefRegex(name, value) if self.__quiet: return - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) @@ -452,7 +453,8 @@ class Transmitter: elif command[1] == "ignorecache": return self.__server.getIgnoreCache(name) elif command[1] == "prefregex": - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index bb42b7a0..2cfaff77 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -264,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) + def testSplitWithOptions(self): + # covering all separators - new-line and spaces: + for sep in ('\n', '\t', ' '): + self.assertEqual(splitWithOptions('a%sb' % (sep,)), ['a', 'b']) + self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)), ['a[x=y]', 'b']) + self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b']) + self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)), ['a[x="y z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b']) + def testIncorrectJail(self): jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 9de1b85a..2a38daa6 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -888,7 +888,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: '," " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index a7ddd6f8..93bf0b14 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -17,3 +17,8 @@ Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SM # failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" } +Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) +# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } +Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index f69e4531..99c1877c 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -103,3 +103,7 @@ Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52 # failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } +Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 +# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 0fe64631..85347c6f 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -557,6 +557,9 @@ class Transmitter(TransmitterBase): jail=self.jailName) self.setGetTest("ignorecache", '', None, jail=self.jailName) + def testJailPrefRegex(self): + self.setGetTest("prefregex", "^Test", jail=self.jailName) + def testJailRegex(self): self.jailAddDelRegexTest("failregex", [