From ad51fb7e1ea42c795135fe97970c31cfce51e446 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Aug 2020 16:41:22 +0200 Subject: [PATCH 1/9] partial cherry-pick fd25c4cbb813db2562e5a2e54d4510eac291a897 (#2768) --- fail2ban/server/failmanager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index eee979fd..3458aed5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value From 1e3da21c680ac9a282f72348869e8cdb3d1e0399 Mon Sep 17 00:00:00 2001 From: TorontoMedia <10255876+TorontoMedia@users.noreply.github.com> Date: Sun, 28 Jun 2020 12:58:41 -0400 Subject: [PATCH 2/9] Remove duplicate method and rename invalid parameter (cherry picked from commit fd25c4cbb813db2562e5a2e54d4510eac291a897) --- fail2ban/server/failmanager.py | 4 ---- fail2ban/server/observer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index bc09c0e2..97b4c7a4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c19549ba..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: From 1707560df8033341e4fb8a2e79db3ea684b42386 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Wed, 26 Feb 2020 10:41:55 +0100 Subject: [PATCH 3/9] Enhance Guacamole jail --- config/filter.d/guacamole.conf | 50 ++++++++++++++++++++++------- config/jail.conf | 1 + fail2ban/tests/files/logs/guacamole | 5 +++ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 09b4e7b0..bc6dbea9 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -5,21 +5,47 @@ [Definition] -# Option: failregex -# Notes.: regex to match the password failures messages in the logfile. -# Values: TEXT -# +logging = catalina +failregex = /failregex> +maxlines = /maxlines> +datepattern = /datepattern> + +[L_catalina] + failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" failed\.$ -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# -ignoreregex = - -# "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p ^WARNING:()** - {^LN-BEG} \ No newline at end of file + {^LN-BEG} + +[L_webapp] + +failregex = ^ \[\S+\] WARN \S+ - Authentication attempt from for user "[^"]+" failed. + +maxlines = 1 + +datepattern = ^%%H:%%M:%%S.%%f + +# DEV Notes: +# +# failregex is based on the default pattern given in Guacamole documentation : +# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging +# +# The following logback.xml Guacamole configuration file can then be used accordingly : +# +# +# /var/log/guacamole.log +# +# /var/log/guacamole.%d.log.gz +# 32 +# +# +# %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n +# +# +# +# +# +# diff --git a/config/jail.conf b/config/jail.conf index 39ebbe6d..40827707 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -440,6 +440,7 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole index 3de67454..ebb7afb0 100644 --- a/fail2ban/tests/files/logs/guacamole +++ b/fail2ban/tests/files/logs/guacamole @@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" } WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. + +# filterOptions: {"logging": "webapp"} + +# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" } +12:57:32.907 [http-nio-8080-exec-10] WARN o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed. From 7b05c1ce7a487f2c269815f8a17b7444213ae555 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 14:52:22 +0200 Subject: [PATCH 4/9] do type-convert only in getCombined (otherwise int/bool conversion prevents substitution or section-related interpolation of tags) --- fail2ban/client/actionreader.py | 11 ++---- fail2ban/client/configreader.py | 54 ++++++++++++++++++++------ fail2ban/tests/clientreadertestcase.py | 7 ++-- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 131e37cb..011a213d 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,17 +38,17 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], - "actionrepair_on_unban": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): @@ -83,11 +83,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 20709b72..f9a6d288 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -228,7 +228,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} @@ -244,12 +244,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue + if convert: + if opttype == "bool": + v = self.getboolean(sec, optname) + if v is None: continue + elif opttype == "int": + v = self.getint(sec, optname) + if v is None: continue + else: + v = self.get(sec, optname, vars=pOptions) else: v = self.get(sec, optname, vars=pOptions) values[optname] = v @@ -267,7 +270,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue @@ -324,8 +327,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -346,10 +350,34 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, pOptions={}): + options = self._configOpts + for optname in options: + if isinstance(options, (list,tuple)): + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + else: + opttype, optvalue = options[optname] + if optname in pOptions: + continue + try: + if opttype == "bool": + v = opts.get(optname) + if v is None or isinstance(v, bool): continue + v = _as_bool(v) + opts[optname] = v + elif opttype == "int": + v = opts.get(optname) + if v is None or isinstance(v, (int, long)): continue + v = int(v) + opts[optname] = v + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -384,6 +412,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 2c1d0a0e..ae39c157 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -490,7 +490,9 @@ class FilterReaderTest(unittest.TestCase): self.__share_cfg = {} def testConvert(self): - output = [['multi-set', 'testcase01', 'addfailregex', [ + output = [ + ['set', 'testcase01', 'maxlines', 1], + ['multi-set', 'testcase01', 'addfailregex', [ "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" @@ -512,7 +514,6 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addjournalmatch', "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], ['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"], - ['set', 'testcase01', 'maxlines', 1], # Last for overide test ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) @@ -529,7 +530,7 @@ class FilterReaderTest(unittest.TestCase): filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - output[-1][-1] = "5" + output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) def testFilterReaderSubstitionDefault(self): From d9b8796792ff2d876fe02e5a469767d33469c276 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:01:34 +0200 Subject: [PATCH 5/9] amend with better (common) handling, documentation and tests --- fail2ban/client/configreader.py | 93 +++++++++++++------------- fail2ban/tests/clientreadertestcase.py | 30 +++++++-- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index f9a6d288..d8817bed 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -233,29 +257,17 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if convert: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue - else: - v = self.get(sec, optname, vars=pOptions) - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -350,33 +362,20 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) - def convertOptions(self, opts, pOptions={}): - options = self._configOpts - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] - if optname in pOptions: - continue - try: - if opttype == "bool": - v = opts.get(optname) - if v is None or isinstance(v, bool): continue - v = _as_bool(v) - opts[optname] = v - elif opttype == "int": - v = opts.get(optname) - if v is None or isinstance(v, (int, long)): continue - v = int(v) - opts[optname] = v - except ValueError: - logSys.warning("Wrong %s value %r for %r. Using default one: %r", - opttype, v, optname, optvalue) - opts[optname] = optvalue + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init @@ -413,7 +412,7 @@ class DefinitionInitConfigReader(ConfigReader): if not opts: raise ValueError('recursive tag definitions unable to be resolved') # convert options after all interpolations: - self.convertOptions(opts) + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ae39c157..8abfd4a5 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -87,6 +87,21 @@ option = %s self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + def testConvert(self): + self.c.add_section("Definition") + self.c.set("Definition", "a", "1") + self.c.set("Definition", "b", "1") + self.c.set("Definition", "c", "test") + opts = self.c.getOptions("Definition", + (('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + opts = self.c.getOptions("Definition", + (('int', 'a'), ('bool', 'b'), ('int', 'c'))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None}) + opts = self.c.getOptions("Definition", + {'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)}) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + def testInaccessibleFile(self): f = os.path.join(self.d, "d.conf") # inaccessible file self._write('d.conf', 0) @@ -483,11 +498,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertRaises(NoSectionError, c.getOptions, 'test', {}) -class FilterReaderTest(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(FilterReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} +class FilterReaderTest(LogCaptureTestCase): def testConvert(self): output = [ @@ -533,6 +544,15 @@ class FilterReaderTest(unittest.TestCase): output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) + def testConvertOptions(self): + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '', 'test': 'X'}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined(); + self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines' + self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:") + def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {}, From 2945fe8cbd39b6e02ae8065878ec2dffc68f0021 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:25:32 +0200 Subject: [PATCH 6/9] changelog --- ChangeLog | 3 +++ fail2ban/client/configreader.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59624d42..cacfdee7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* fixed type conversion in config readers (take place after all interpolations get ready), that allows to + specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters. * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing @@ -61,6 +63,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### New Features and Enhancements * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d8817bed..1b5a56a2 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -282,7 +282,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue From 5a2cc4e1c5219645ca801c8ba90bab5c609340ec Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:05:20 +0200 Subject: [PATCH 7/9] substituteRecursiveTags: more precise self- or cyclic-recursion prevention (don't clear replacement counts of tags, rather consider replacement count by tax X in tag Y) --- fail2ban/helpers.py | 14 ++++++++------ fail2ban/tests/actiontestcase.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 3ef7d543..dc7852ae 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -398,8 +398,8 @@ def splitWithOptions(option): # tags () in tagged options. # -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 +# max tag replacement count (considering tag X in tag Y repeat): +MAX_TAG_REPLACE_COUNT = 25 # compiled RE for tag name (replacement name) TAG_CRE = re.compile(r'<([^ <>]+)>') @@ -433,6 +433,7 @@ def substituteRecursiveTags(inptags, conditional='', done = set() noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) + repCounts = {} while True: repFlag = False # substitute each value: @@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='', value = orgval = uni_string(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) - refCounts = {} + rplc = repCounts.get(tag, {}) #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: # found replacement tag: @@ -454,13 +455,13 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue #logSys.log(5, 'found: %s' % rtag) - if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: + if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, rtag, refCounts, value)) + (tag, rtag, rplc, value)) repl = None if conditional: repl = tags.get(rtag + '?' + conditional) @@ -480,7 +481,7 @@ def substituteRecursiveTags(inptags, conditional='', value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[rtag] = refCounts.get(rtag, 0) + 1 + rplc[rtag] = rplc.get(rtag, 0) + 1 # the next match for replace: m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) @@ -488,6 +489,7 @@ def substituteRecursiveTags(inptags, conditional='', if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): if tre_search(value): + repCounts[tag] = rplc repFlag = True # copy return tags dict to prevent modifying of inptags: if id(tags) == id(inptags): diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 3425667c..125706af 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase): delattr(self.__action, 'ac') # produce self-referencing query except: self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", - lambda: self.__action.replaceTag(">>>>>>>>>>>>>>>>>>>>", + lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) From e569281d6b9880fdf3d8084dae295880f6700897 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:08:04 +0200 Subject: [PATCH 8/9] avoids overwrite of `known/option` with unmodified (not available) value of `option` from .local config file, so it wouldn't cause self-recursion if `option` already has a reference to `known/option` (from some include) in .conf file; closes gh-2751 --- fail2ban/client/configparserinc.py | 7 ++++--- fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/files/filter.d/testcase02.conf | 12 ++++++++++++ fail2ban/tests/files/filter.d/testcase02.local | 4 ++++ 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 fail2ban/tests/files/filter.d/testcase02.conf create mode 100644 fail2ban/tests/files/filter.d/testcase02.local diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index e0f39579..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -372,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 8abfd4a5..bb42b7a0 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -562,6 +562,17 @@ class FilterReaderTest(LogCaptureTestCase): c = filterReader.convert() self.assertSortedEqual(c, output) + def testFilterReaderSubstKnown(self): + # testcase02.conf + testcase02.local, test covering that known/option is not overridden + # with unmodified (not available) value of option from .local config file, so wouldn't + # cause self-recursion if option already has a reference to known/option in .conf file. + filterReader = FilterReader('testcase02', "jailname", {}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined() + self.assertTrue('sshd' in opts['failregex']) + def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, diff --git a/fail2ban/tests/files/filter.d/testcase02.conf b/fail2ban/tests/files/filter.d/testcase02.conf new file mode 100644 index 00000000..99b3bb45 --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.conf @@ -0,0 +1,12 @@ +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + +[Definition] + +_daemon = sshd +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? + +failregex = %(__prefix_line)s test \ No newline at end of file diff --git a/fail2ban/tests/files/filter.d/testcase02.local b/fail2ban/tests/files/filter.d/testcase02.local new file mode 100644 index 00000000..bfc81d4b --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.local @@ -0,0 +1,4 @@ +[Definition] + +# no options here, coverage for testFilterReaderSubstKnown: +# avoid to overwrite known/option with unmodified (not available) value of option from .local config file \ No newline at end of file From be3115cda0c5ab7def871256995974880221683d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 13:31:29 +0200 Subject: [PATCH 9/9] fix year overflow (9999) by format of datetime (time2str for end of ban of persistent ticket); closes gh-2817 --- fail2ban/server/mytime.py | 7 +++++-- fail2ban/tests/banmanagertestcase.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 98b69bd4..e4b091a7 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -121,8 +121,11 @@ class MyTime: @return ISO-capable string representation of given unixTime """ - return datetime.datetime.fromtimestamp( - unixTime).replace(microsecond=0).strftime(format) + # consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ) + dt = datetime.datetime.fromtimestamp( + unixTime).replace(microsecond=0 + ) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59) + return dt.strftime(format) ## precreate/precompile primitives used in str2seconds: diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a5b37ef6..3be31bc5 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -154,6 +154,21 @@ class AddFailure(unittest.TestCase): finally: self.__banManager.setBanTime(btime) + def testBanList(self): + tickets = [ + BanTicket('192.0.2.1', 1167605999.0), + BanTicket('192.0.2.2', 1167605999.0), + ] + tickets[1].setBanTime(-1) + for t in tickets: + self.__banManager.addBanTicket(t) + self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True), + [ + '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59', + '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59' + ] + ) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self):