diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 55ceda93..0fd55f41 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,6 +28,7 @@ import os from .configreader import DefinitionInitConfigReader from ..helpers import getLogger +from ..server.action import CommandAction # Gets the instance of the logger. logSys = getLogger(__name__) @@ -69,7 +70,8 @@ class ActionReader(DefinitionInitConfigReader): return self._name def convert(self): - opts = self.getCombined(ignore=('timeout', 'bantime')) + opts = self.getCombined( + ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) # type-convert only after combined (otherwise boolean converting prevents substitution): if opts.get('norestored'): opts['norestored'] = self._convert_to_boolean(opts['norestored']) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 8098433e..2caa97ce 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -29,8 +29,7 @@ import os from ConfigParser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel -from ..helpers import getLogger -from ..server.action import CommandAction +from ..helpers import getLogger, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) @@ -328,6 +327,11 @@ class DefinitionInitConfigReader(ConfigReader): return value.lower() in ("1", "yes", "true", "on") def getCombOption(self, optname): + """Get combined definition option (as string) using pre-set and init + options as preselection (values with higher precedence as specified in section). + + Can be used only after calling of getOptions. + """ try: return self._defCache[optname] except KeyError: @@ -352,7 +356,7 @@ class DefinitionInitConfigReader(ConfigReader): n, cond = cond.groups() ignore.add(n) # substiture options already specified direct: - opts = CommandAction.substituteRecursiveTags(combinedopts, + opts = substituteRecursiveTags(combinedopts, ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 4f4426b0..1e5a053b 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -200,6 +200,105 @@ else: raise return uni_decode(x, enc, 'replace') + +# +# Following facilities used for safe recursive interpolation of +# tags () in tagged options. +# + +# max tag replacement count: +MAX_TAG_REPLACE_COUNT = 10 + +# compiled RE for tag name (replacement name) +TAG_CRE = re.compile(r'<([^ <>]+)>') + +def substituteRecursiveTags(inptags, conditional='', + ignore=(), addrepl=None +): + """Sort out tag definitions within other tags. + Since v.0.9.2 supports embedded interpolation (see test cases for examples). + + so: becomes: + a = 3 a = 3 + b = _3 b = 3_3 + + Parameters + ---------- + inptags : dict + Dictionary of tags(keys) and their values. + + Returns + ------- + dict + Dictionary of tags(keys) and their values, with tags + within the values recursively replaced. + """ + # copy return tags dict to prevent modifying of inptags: + tags = inptags.copy() + t = TAG_CRE + ignore = set(ignore) + done = set() + # repeat substitution while embedded-recursive (repFlag is True) + while True: + repFlag = False + # substitute each value: + for tag in tags.iterkeys(): + # ignore escaped or already done (or in ignore list): + if tag in ignore or tag in done: continue + value = orgval = str(tags[tag]) + # search and replace all tags within value, that can be interpolated using other tags: + m = t.search(value) + refCounts = {} + #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) + while m: + found_tag = m.group(1) + # don't replace tags that should be currently ignored (pre-replacement): + if found_tag in ignore: + m = t.search(value, m.end()) + continue + #logSys.log(5, 'found: %s' % found_tag) + if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT: + # recursive definitions are bad + #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) + raise ValueError( + "properties contain self referencing definitions " + "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % + (tag, found_tag, refCounts, value)) + repl = None + if conditional: + repl = tags.get(found_tag + '?' + conditional) + if repl is None: + repl = tags.get(found_tag) + # try to find tag using additional replacement (callable): + if repl is None and addrepl is not None: + repl = addrepl(found_tag) + if repl is None: + # Missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) + continue + value = value.replace('<%s>' % found_tag, repl) + #logSys.log(5, 'value now: %s' % value) + # increment reference count: + refCounts[found_tag] = refCounts.get(found_tag, 0) + 1 + # the next match for replace: + m = t.search(value, m.start()) + #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) + # was substituted? + if orgval != value: + # check still contains any tag - should be repeated (possible embedded-recursive substitution): + if t.search(value): + repFlag = True + tags[tag] = value + # no more sub tags (and no possible composite), add this tag to done set (just to be faster): + if '<' not in value: done.add(tag) + # stop interpolation, if no replacements anymore: + if not repFlag: + break + return tags + + class BgService(object): """Background servicing diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d14c9aab..2f7f423c 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -23,7 +23,6 @@ __license__ = "GPL" import logging import os -import re import signal import subprocess import tempfile @@ -35,7 +34,7 @@ from collections import MutableMapping from .ipdns import asip from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger +from ..helpers import getLogger, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) @@ -46,12 +45,6 @@ _cmd_lock = threading.Lock() # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`: allowed_ipv6 = True -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 - -# compiled RE for tag name (replacement name) -TAG_CRE = re.compile(r'<([^ <>]+)>') - class CallingMap(MutableMapping): """A Mapping type which returns the result of callable values. @@ -364,93 +357,6 @@ class CommandAction(ActionBase): """ return self._executeOperation('', 'reloading') - @classmethod - def substituteRecursiveTags(cls, inptags, conditional='', - ignore=(), addrepl=None - ): - """Sort out tag definitions within other tags. - Since v.0.9.2 supports embedded interpolation (see test cases for examples). - - so: becomes: - a = 3 a = 3 - b = _3 b = 3_3 - - Parameters - ---------- - inptags : dict - Dictionary of tags(keys) and their values. - - Returns - ------- - dict - Dictionary of tags(keys) and their values, with tags - within the values recursively replaced. - """ - # copy return tags dict to prevent modifying of inptags: - tags = inptags.copy() - t = TAG_CRE - ignore = set(ignore) - done = cls._escapedTags.copy() | ignore - # repeat substitution while embedded-recursive (repFlag is True) - while True: - repFlag = False - # substitute each value: - for tag in tags.iterkeys(): - # ignore escaped or already done (or in ignore list): - if tag in done: continue - value = orgval = str(tags[tag]) - # search and replace all tags within value, that can be interpolated using other tags: - m = t.search(value) - refCounts = {} - #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) - while m: - found_tag = m.group(1) - # don't replace tags that should be currently ignored (pre-replacement): - if found_tag in ignore: - m = t.search(value, m.end()) - continue - #logSys.log(5, 'found: %s' % found_tag) - if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT: - # recursive definitions are bad - #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) - raise ValueError( - "properties contain self referencing definitions " - "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, found_tag, refCounts, value)) - repl = None - if found_tag not in cls._escapedTags: - if conditional: - repl = tags.get(found_tag + '?' + conditional) - if repl is None: - repl = tags.get(found_tag) - if repl is None and addrepl is not None: - repl = addrepl(found_tag) - if repl is None: - # Escaped or missing tags - just continue on searching after end of match - # Missing tags are ok - cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.end()) - continue - value = value.replace('<%s>' % found_tag, repl) - #logSys.log(5, 'value now: %s' % value) - # increment reference count: - refCounts[found_tag] = refCounts.get(found_tag, 0) + 1 - # the next match for replace: - m = t.search(value, m.start()) - #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - # was substituted? - if orgval != value: - # check still contains any tag - should be repeated (possible embedded-recursive substitution): - if t.search(value): - repFlag = True - tags[tag] = value - # no more sub tags (and no possible composite), add this tag to done set (just to be faster): - if '<' not in value: done.add(tag) - # stop interpolation, if no replacements anymore: - if not repFlag: - break - return tags - @staticmethod def escapeTag(value): """Escape characters which may be used for command injection. @@ -501,7 +407,7 @@ class CommandAction(ActionBase): return string # replace: string = query - aInfo = cls.substituteRecursiveTags(aInfo, conditional) + aInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) for tag in aInfo: if "<%s>" % tag in query: value = aInfo.get(tag + '?' + conditional) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 403f7ea6..8a74201c 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -29,7 +29,7 @@ import tempfile import time import unittest -from ..server.action import CommandAction, CallingMap +from ..server.action import CommandAction, CallingMap, substituteRecursiveTags from ..server.actions import OrderedDict from ..server.utils import Utils @@ -56,30 +56,30 @@ class CommandActionTest(LogCaptureTestCase): } # Recursion is bad self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': ''})) + lambda: substituteRecursiveTags({'A': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': '', 'B': ''})) + lambda: substituteRecursiveTags({'A': '', 'B': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) + lambda: substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) # Unresolveable substition self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': 'to= fromip=', 'C': '', 'B': '', 'D': ''})) + lambda: substituteRecursiveTags({'A': 'to= fromip=', 'C': '', 'B': '', 'D': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) + lambda: substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) # We need here an ordered, because the sequence of iteration is very important for this test if OrderedDict: # No cyclic recursion, just multiple replacement of tag , should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict( + self.assertEqual(substituteRecursiveTags( OrderedDict( (('X', 'x=x'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) ), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'} ) # No cyclic recursion, just multiple replacement of tag in composite tags, should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict( + self.assertEqual(substituteRecursiveTags( OrderedDict( (('X', 'x=x <> <>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) ), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'} ) # No cyclic recursion, just multiple replacement of same tags, should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertEqual(substituteRecursiveTags( OrderedDict(( ('actionstart', 'ipset create hash:ip timeout family \n -I '), ('ipmset', 'f2b-'), ('name', 'any'), @@ -111,42 +111,42 @@ class CommandActionTest(LogCaptureTestCase): )) ) # Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle: - self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( ('A', '<>'), ('B', 'D'), ('C', 'E'), ('DE', 'cycle '), )) )) - self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( ('DE', 'cycle '), ('A', '<>'), ('B', 'D'), ('C', 'E'), )) )) # missing tags are ok - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ''}), {'A': ''}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(substituteRecursiveTags({'A': ''}), {'A': ''}) + self.assertEqual(substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) + self.assertEqual(substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Escaped tags should be ignored - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Multiple stuff on same line is ok - self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), + self.assertEqual(substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), { 'failregex': "to=pokie fromip= evilperson=pokie", 'honeypot': 'pokie', 'ignoreregex': '', }) # rest is just cool - self.assertEqual(CommandAction.substituteRecursiveTags(aInfo), + self.assertEqual(substituteRecursiveTags(aInfo), { 'HOST': "192.0.2.0", 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', }) # obscure embedded case - self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), + self.assertEqual(substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), {'A': '', 'PREF': 'IPV4'}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), + self.assertEqual(substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) # more embedded within a string and two interpolations - self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), + self.assertEqual(substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self):