From 5ce8d4f741768c59fef5216323e9937ffb44f885 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Apr 2017 12:11:17 +0200 Subject: [PATCH 1/6] fixes default backend handling (as default used value of `known/backend`, which can now be overridden in default section of jail.local); introduces fallback for `known/option`: interpolate missing `known/option` as `option` from default section --- config/paths-common.conf | 2 +- fail2ban/client/configparserinc.py | 56 ++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/config/paths-common.conf b/config/paths-common.conf index 9072136c..522957fe 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -7,7 +7,7 @@ after = paths-overrides.local [DEFAULT] -default_backend = auto +default_backend = %(known/backend)s sshd_log = %(syslog_authpriv)s sshd_backend = %(default_backend)s diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 6de513cd..7d63682a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -32,8 +32,8 @@ from ..helpers import getLogger if sys.version_info >= (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) - from configparser import ConfigParser as SafeConfigParser, NoSectionError, \ - BasicInterpolation + from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ + InterpolationMissingOptionError, NoSectionError # And interpolation of __name__ was simply removed, thus we need to # decorate default interpolator to handle it @@ -52,20 +52,43 @@ if sys.version_info >= (3,2): But should be fine to reincarnate for our use case """ def _interpolate_some(self, parser, option, accum, rest, section, map, - depth): + *args, **kwargs): if section and not (__name__ in map): map = map.copy() # just to be safe map['__name__'] = section - return super(BasicInterpolationWithName, self)._interpolate_some( - parser, option, accum, rest, section, map, depth) + try: + return super(BasicInterpolationWithName, self)._interpolate_some( + parser, option, accum, rest, section, map, *args, **kwargs) + except InterpolationMissingOptionError as e: + # fallback: try to wrap missing default options as "known/options": + if not parser._map_known_defaults(section, option, rest, map): # pragma: no cover + raise e + # try again: + return super(BasicInterpolationWithName, self)._interpolate_some( + parser, option, accum, rest, section, map, *args, **kwargs) else: # pragma: no cover - from ConfigParser import SafeConfigParser, NoSectionError + from ConfigParser import SafeConfigParser, \ + InterpolationMissingOptionError, NoSectionError + + # Interpolate missing known/option as option from default section + SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some + def _interpolate_some(self, option, accum, rest, section, map, *args, **kwargs): + try: + return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) + except InterpolationMissingOptionError as e: + # fallback: try to wrap missing default options as "known/options": + if self._map_known_defaults(section, option, rest, map): # pragma: no cover + raise e + # try again: + return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) + SafeConfigParser._interpolate_some = _interpolate_some # Gets the instance of the logger. logSys = getLogger(__name__) logLevel = 7 + __all__ = ['SafeConfigParserWithIncludes'] @@ -100,6 +123,8 @@ after = 1.conf SECTION_NAME = "INCLUDES" + _KNOWN_OPTSUBST_CRE = re.compile(r'%\(known/([^\)]+)\)s') + CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") if sys.version_info >= (3,2): @@ -117,6 +142,25 @@ after = 1.conf SafeConfigParser.__init__(self, *args, **kwargs) self._cfg_share = share_config + def _map_known_defaults(self, section, option, rest, map): + """ Fallback: try to wrap missing default options as "known/options" + """ + known = SafeConfigParserWithIncludes._KNOWN_OPTSUBST_CRE.findall(rest) + if not known: # pragma: no cover + return 0 + for opt in known: + kopt = 'known/'+opt + if kopt not in map: + try: + v = self._defaults[opt] + except KeyError: + continue + self._defaults[kopt] = v + try: # for python 2.6 we should duplicate it in map-vars also: + map[kopt] = v + except: pass + return 1 + @property def share_config(self): return self._cfg_share From e52f483557f5be7416a9fa4bb4c5f71bf9ea4272 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Apr 2017 15:20:35 +0200 Subject: [PATCH 2/6] Config reader's: introduced new syntax `%(section/option)s`, in opposite to extended interpolation of python 3 `${section:option}` work with all supported python version in fail2ban and this syntax is like our another features like `%(known/option)s`, etc.; Variable `default_backend` switched to `%(default/backend)s`, so totally backwards compatible now, but now the setting of parameter `backend` in default section of `jail.local` can overwrite default backend also. Test cases extended: test targeted section options "section/option" (default and cross sections options); --- config/paths-common.conf | 2 +- fail2ban/client/configparserinc.py | 108 ++++++++++++++++--------- fail2ban/client/configreader.py | 2 +- fail2ban/tests/clientreadertestcase.py | 27 +++++++ 4 files changed, 97 insertions(+), 42 deletions(-) diff --git a/config/paths-common.conf b/config/paths-common.conf index 522957fe..51323d6b 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -7,7 +7,7 @@ after = paths-overrides.local [DEFAULT] -default_backend = %(known/backend)s +default_backend = %(default/backend)s sshd_log = %(syslog_authpriv)s sshd_backend = %(default_backend)s diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7d63682a..1ac980e0 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -56,14 +56,8 @@ if sys.version_info >= (3,2): if section and not (__name__ in map): map = map.copy() # just to be safe map['__name__'] = section - try: - return super(BasicInterpolationWithName, self)._interpolate_some( - parser, option, accum, rest, section, map, *args, **kwargs) - except InterpolationMissingOptionError as e: - # fallback: try to wrap missing default options as "known/options": - if not parser._map_known_defaults(section, option, rest, map): # pragma: no cover - raise e - # try again: + # try to wrap section options like %(section/option)s: + parser._map_section_options(section, option, rest, map) return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) @@ -74,16 +68,16 @@ else: # pragma: no cover # Interpolate missing known/option as option from default section SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some def _interpolate_some(self, option, accum, rest, section, map, *args, **kwargs): - try: - return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) - except InterpolationMissingOptionError as e: - # fallback: try to wrap missing default options as "known/options": - if self._map_known_defaults(section, option, rest, map): # pragma: no cover - raise e - # try again: - return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) + # try to wrap section options like %(section/option)s: + self._map_section_options(section, option, rest, map) + return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) SafeConfigParser._interpolate_some = _interpolate_some +try: + from configparser import _UNSET +except ImportError: + _UNSET = object() + # Gets the instance of the logger. logSys = getLogger(__name__) logLevel = 7 @@ -123,7 +117,8 @@ after = 1.conf SECTION_NAME = "INCLUDES" - _KNOWN_OPTSUBST_CRE = re.compile(r'%\(known/([^\)]+)\)s') + SECTION_OPT_CRE = re.compile(r'^(\w+)/(.+)$') + SECTION_OPTSUBST_CRE = re.compile(r'%\((\w+/([^\)]+))\)s') CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") @@ -142,22 +137,43 @@ after = 1.conf SafeConfigParser.__init__(self, *args, **kwargs) self._cfg_share = share_config - def _map_known_defaults(self, section, option, rest, map): - """ Fallback: try to wrap missing default options as "known/options" + def _map_section_options(self, section, option, rest, map): """ - known = SafeConfigParserWithIncludes._KNOWN_OPTSUBST_CRE.findall(rest) - if not known: # pragma: no cover + Interpolates values of the section options (name syntax `%(section/option)s`). + + Fallback: try to wrap missing default options as "default/options" resp. "known/options" + """ + if '/' not in rest or '%(' not in rest: # pragma: no cover return 0 - for opt in known: - kopt = 'known/'+opt - if kopt not in map: - try: - v = self._defaults[opt] - except KeyError: - continue - self._defaults[kopt] = v - try: # for python 2.6 we should duplicate it in map-vars also: - map[kopt] = v + soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest) + if not soptrep: # pragma: no cover + return 0 + for sopt, opt in soptrep: + if sopt not in map: + sec = sopt[:~len(opt)] + seclwr = sec.lower() + if seclwr != 'default': + if seclwr == 'known': + # try get raw value from known options: + try: + v = self._sections['KNOWN'][opt] + except KeyError: + # fallback to default: + try: + v = self._defaults[opt] + except KeyError: + continue + else: + # get substituted value of opt in section: + v = self.get(sec, opt) + else: + try: + v = self._defaults[opt] + except KeyError: + continue + self._defaults[sopt] = v + try: # for some python versions need to duplicate it in map-vars also: + map[sopt] = v except: pass return 1 @@ -244,6 +260,18 @@ after = 1.conf def get_sections(self): return self._sections + def get(self, sec, opt, raw=False, vars={}, fallback=_UNSET): + try: + return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars) + except: + sopt = SafeConfigParserWithIncludes.SECTION_OPT_CRE.match(opt) + if not sopt: raise + sec, opt = sopt.groups() + if sec.lower() == 'default': + # get default raw value: + return self._defaults[opt] + return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars) + def options(self, section, withDefault=True): """Return a list of option names for the given section name. @@ -303,11 +331,7 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - sk = {} - for k, v in s2.iteritems(): - if not k.startswith('known/') and k != '__name__': - sk['known/'+k] = v - s2.update(sk) + self.merge_section('KNOWN', s2, '') # merge section s2.update(s) else: @@ -324,14 +348,18 @@ after = 1.conf else: return SafeConfigParser.read(self, fileNamesFull) - def merge_section(self, section, options, pref='known/'): + def merge_section(self, section, options, pref=None): alls = self.get_sections() - if pref == '': - alls[section].update(options) + try: + sec = alls[section] + except KeyError: + alls[section] = sec = dict() + if not pref: + sec.update(options) return sk = {} for k, v in options.iteritems(): if not k.startswith(pref) and k != '__name__': sk[pref+k] = v - alls[section].update(sk) + sec.update(sk) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index bbc18384..381af759 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -110,7 +110,7 @@ class ConfigReader(): def sections(self): try: - return self._cfg.sections() + return (n for n in self._cfg.sections() if n != 'KNOWN') except AttributeError: return [] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 6cc2f659..325a1d68 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -162,6 +162,33 @@ c = d ;in line comment self.assertEqual(self.c.get('DEFAULT', 'b'), 'a') self.assertEqual(self.c.get('DEFAULT', 'c'), 'd') + def testTargetedSectionOptions(self): + self.assertFalse(self.c.read('g')) # nothing is there yet + self._write("g.conf", value=None, content=""" +[DEFAULT] +a = def-a +b = def-b,a:`%(a)s` +c = def-c,b:"%(b)s" +d = def-d-b:"%(known/b)s" +[jail] +a = jail-a-%(test/a)s +b = jail-b-%(test/b)s +[test] +a = test-a-%(default/a)s +b = test-b-%(known/b)s +""") + self.assertTrue(self.c.read('g')) + self.assertEqual(self.c.get('test', 'a'), 'test-a-def-a') + self.assertEqual(self.c.get('test', 'b'), 'test-b-def-b,a:`test-a-def-a`') + self.assertEqual(self.c.get('jail', 'a'), 'jail-a-test-a-def-a') + self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`test-a-def-a`') + self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"') + self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"') + self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"') + class JailReaderTest(LogCaptureTestCase): From 0782b28460057a913ac7ef7bf2fcf6bdc2debc58 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Aug 2017 21:13:49 +0200 Subject: [PATCH 3/6] allow usage of "-" character in the name of sections by extended interpolation (like `%(my-section/some-value)s`). --- fail2ban/client/configparserinc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 1ac980e0..cc245b0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -117,8 +117,8 @@ after = 1.conf SECTION_NAME = "INCLUDES" - SECTION_OPT_CRE = re.compile(r'^(\w+)/(.+)$') - SECTION_OPTSUBST_CRE = re.compile(r'%\((\w+/([^\)]+))\)s') + SECTION_OPT_CRE = re.compile(r'^([\w\-]+)/(.+)$') + SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s') CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") From 9a768c19d0b3f7039584d640de9fd1322072517b Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Aug 2017 21:48:50 +0200 Subject: [PATCH 4/6] Fixes raw to interpolated value by recursive substitution. Avoid too deeply recursive extended interpolation cross sections (test covered now); --- fail2ban/client/configparserinc.py | 4 ++-- fail2ban/tests/clientreadertestcase.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index cc245b0a..7cc9b9fa 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -164,8 +164,8 @@ after = 1.conf except KeyError: continue else: - # get substituted value of opt in section: - v = self.get(sec, opt) + # get raw value of opt in section: + v = self.get(sec, opt, raw=True) else: try: v = self._defaults[opt] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 325a1d68..c3a10c36 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -170,24 +170,31 @@ a = def-a b = def-b,a:`%(a)s` c = def-c,b:"%(b)s" d = def-d-b:"%(known/b)s" + [jail] a = jail-a-%(test/a)s b = jail-b-%(test/b)s +y = %(test/y)s + [test] a = test-a-%(default/a)s b = test-b-%(known/b)s +x = %(test/x)s +y = %(jail/y)s """) self.assertTrue(self.c.read('g')) self.assertEqual(self.c.get('test', 'a'), 'test-a-def-a') self.assertEqual(self.c.get('test', 'b'), 'test-b-def-b,a:`test-a-def-a`') self.assertEqual(self.c.get('jail', 'a'), 'jail-a-test-a-def-a') - self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`test-a-def-a`') - self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`jail-a-test-a-def-a`') + self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"') self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"') self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"') + self.assertRaises(Exception, self.c.get, 'test', 'x') + self.assertRaises(Exception, self.c.get, 'jail', 'y') class JailReaderTest(LogCaptureTestCase): From 6f4fde2c29743286907fec5c5afc5fb11122401f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Aug 2017 21:50:38 +0200 Subject: [PATCH 5/6] Update changelog and man/jail.conf.5 --- ChangeLog | 7 +++++++ man/jail.conf.5 | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 32f7b521..2672889a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -134,6 +134,13 @@ TODO: implementing of options resp. other tasks from PR #1346 Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone abbreviations. * `filter.d/courier-auth.conf`: support failed logins with method only +* Config reader's: introduced new syntax `%(section/option)s`, in opposite to extended interpolation of + python 3 `${section:option}` work with all supported python version in fail2ban and this syntax is + like our another features like `%(known/option)s`, etc. (gh-1750) +* Variable `default_backend` switched to `%(default/backend)s`, so totally backwards compatible now, + but now the setting of parameter `backend` in default section of `jail.local` can overwrite default + backend also (see gh-1750). In the future versions parameter `default_backend` can be removed (incompatibility, + possibly some distributions affected). ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc diff --git a/man/jail.conf.5 b/man/jail.conf.5 index a8afe53c..e939771b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -90,11 +90,16 @@ indicates that the specified file is to be parsed after the current file. .RE Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. -Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g. + +Fail2ban has more advanced syntax (similar python extended interpolation). This extended interpolation is using \fB%(section/parameter)s\fR to denote a value from a foreign section. +.br +Besides cross section interpolation the value of parameter in \fI[DEFAULT]\fR section can be retrieved with \fB%(default/parameter)s\fR. +.br +Fail2ban supports also another feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g. .RS .nf -baduseragents = IE|wget +baduseragents = IE|wget|%(my-settings/baduseragents)s failregex = %(known/failregex)s useragent=%(baduseragents)s .fi From 7549f44dedb886767eccc19a84061f9d1abd841d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 9 Aug 2017 15:36:36 +0200 Subject: [PATCH 6/6] code review: removed unused (unneeded) code, coverage increase... --- fail2ban/client/configparserinc.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7cc9b9fa..b626be9b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -73,11 +73,6 @@ else: # pragma: no cover return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) SafeConfigParser._interpolate_some = _interpolate_some -try: - from configparser import _UNSET -except ImportError: - _UNSET = object() - # Gets the instance of the logger. logSys = getLogger(__name__) logLevel = 7 @@ -117,7 +112,6 @@ after = 1.conf SECTION_NAME = "INCLUDES" - SECTION_OPT_CRE = re.compile(r'^([\w\-]+)/(.+)$') SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s') CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") @@ -161,7 +155,7 @@ after = 1.conf # fallback to default: try: v = self._defaults[opt] - except KeyError: + except KeyError: # pragma: no cover continue else: # get raw value of opt in section: @@ -169,7 +163,7 @@ after = 1.conf else: try: v = self._defaults[opt] - except KeyError: + except KeyError: # pragma: no cover continue self._defaults[sopt] = v try: # for some python versions need to duplicate it in map-vars also: @@ -260,18 +254,6 @@ after = 1.conf def get_sections(self): return self._sections - def get(self, sec, opt, raw=False, vars={}, fallback=_UNSET): - try: - return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars) - except: - sopt = SafeConfigParserWithIncludes.SECTION_OPT_CRE.match(opt) - if not sopt: raise - sec, opt = sopt.groups() - if sec.lower() == 'default': - # get default raw value: - return self._defaults[opt] - return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars) - def options(self, section, withDefault=True): """Return a list of option names for the given section name. @@ -279,7 +261,7 @@ after = 1.conf """ try: opts = self._sections[section] - except KeyError: + except KeyError: # pragma: no cover raise NoSectionError(section) if withDefault: # mix it with defaults: