From dd8081ade5ace23782efcf0c96b7f8c9f16228a7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 20:00:42 +0200 Subject: [PATCH 1/3] extends capturing alternate tags in filter, implementing new tag prefix `` with all value of tags), for examples see new tests in fail2banregextestcase; closes gh-2755 (extends #1454 and #1698). --- fail2ban/server/action.py | 2 +- fail2ban/server/failregex.py | 56 +++++++++++++++++-------- fail2ban/server/ipdns.py | 2 +- fail2ban/tests/fail2banregextestcase.py | 17 ++++++++ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d0262171..1c313cf0 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -868,7 +868,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 0ae9acc5..9bbf5779 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -87,20 +87,24 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -128,18 +132,23 @@ class Regex: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex self._altValues = {} + self._tupleValues = {} for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n + n = COMPLNAME_CRE.match(k) + if n: + if n.group(1) == ALTNAME_PRE: + self._altValues[k] = mapTag2Opt(n.group(2)) + else: + self._tupleValues[k] = mapTag2Opt(n.group(2)) self._altValues = list(self._altValues.items()) if len(self._altValues) else None + self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) # set fetch handler depending on presence of alternate tags: - self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -284,12 +293,23 @@ class Regex: def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..18e1bd02 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -337,7 +337,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..334fad1c 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -340,6 +340,23 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) self.assertLogged('kevin') self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s* \S+')) + self.assertLogged(str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left 192.0.2.3 right', + r'^\s*\S+ \S+')) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*\S+ : \S+')) + self.assertLogged(str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() # row with id : self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) self.assertLogged("['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) From ec3000798d9b42c3f05d40e810f43ae444c14d97 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 21:25:42 +0200 Subject: [PATCH 2/3] ensure that set of alternate tags or combine tuple tags take place ordered (sort the lists by its name or index) --- fail2ban/server/failregex.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 9bbf5779..4032ecdb 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -131,23 +131,26 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} - self._tupleValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): n = COMPLNAME_CRE.match(k) if n: - if n.group(1) == ALTNAME_PRE: - self._altValues[k] = mapTag2Opt(n.group(2)) + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) else: - self._tupleValues[k] = mapTag2Opt(n.group(2)) - self._altValues = list(self._altValues.items()) if len(self._altValues) else None - self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - # set fetch handler depending on presence of alternate tags: + # set fetch handler depending on presence of alternate (or tuple) tags: self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): From 98983adf761282ba4e6ad4d83a77fffcc6030573 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 17:14:13 +0200 Subject: [PATCH 3/3] update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 266ca614..65568f62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,8 @@ ver. 0.11.2-dev (20??/??/??) - development edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `` with all value of `` tags (gh-2755) ver. 0.11.1 (2020/01/11) - this-is-the-way