diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 2f31d8b9..b9cd0584 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" +actionban = curl --fail --data-urlencode "server=" --data "apikey=" --data "service=" --data "ip=" --data-urlencode "logs=
" --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -64,8 +64,6 @@ actionban = curl --fail --data-urlencode 'server=' --data 'apikey== (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ - InterpolationMissingOptionError, NoSectionError + InterpolationMissingOptionError, NoOptionError, NoSectionError # And interpolation of __name__ was simply removed, thus we need to # decorate default interpolator to handle it @@ -63,7 +63,7 @@ if sys.version_info >= (3,2): else: # pragma: no cover from ConfigParser import SafeConfigParser, \ - InterpolationMissingOptionError, NoSectionError + InterpolationMissingOptionError, NoOptionError, NoSectionError # Interpolate missing known/option as option from default section SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some @@ -112,6 +112,8 @@ after = 1.conf SECTION_NAME = "INCLUDES" + SECTION_OPTNAME_CRE = re.compile(r'^([\w\-]+)/([^\s>]+)$') + SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s') CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") @@ -131,7 +133,36 @@ after = 1.conf SafeConfigParser.__init__(self, *args, **kwargs) self._cfg_share = share_config - def _map_section_options(self, section, option, rest, map): + def get_ex(self, section, option, raw=False, vars={}): + """Get an option value for a given section. + + In opposite to `get`, it differentiate session-related option name like `sec/opt`. + """ + sopt = None + # if option name contains section: + if '/' in option: + sopt = SafeConfigParserWithIncludes.SECTION_OPTNAME_CRE.search(option) + # try get value from named section/option: + if sopt: + sec = sopt.group(1) + opt = sopt.group(2) + seclwr = sec.lower() + if seclwr == 'known': + # try get value firstly from known options, hereafter from current section: + sopt = ('KNOWN/'+section, section) + else: + sopt = (sec,) if seclwr != 'default' else ("DEFAULT",) + for sec in sopt: + try: + v = self.get(sec, opt, raw=raw) + return v + except (NoSectionError, NoOptionError) as e: + pass + # get value of section/option using given section and vars (fallback): + v = self.get(section, option, raw=raw, vars=vars) + return v + + def _map_section_options(self, section, option, rest, defaults): """ Interpolates values of the section options (name syntax `%(section/option)s`). @@ -139,37 +170,54 @@ after = 1.conf """ if '/' not in rest or '%(' not in rest: # pragma: no cover return 0 + rplcmnt = 0 soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest) if not soptrep: # pragma: no cover return 0 for sopt, opt in soptrep: - if sopt not in map: + if sopt not in defaults: sec = sopt[:~len(opt)] seclwr = sec.lower() if seclwr != 'default': + usedef = 0 if seclwr == 'known': # try get raw value from known options: try: v = self._sections['KNOWN/'+section][opt] except KeyError: # fallback to default: - try: - v = self._defaults[opt] - except KeyError: # pragma: no cover - continue + usedef = 1 else: # get raw value of opt in section: - v = self.get(sec, opt, raw=True) + try: + # if section not found - ignore: + try: + sec = self._sections[sec] + except KeyError: # pragma: no cover + continue + v = sec[opt] + except KeyError: # pragma: no cover + # fallback to default: + usedef = 1 else: + usedef = 1 + if usedef: try: v = self._defaults[opt] except KeyError: # pragma: no cover continue - self._defaults[sopt] = v - try: # for some python versions need to duplicate it in map-vars also: - map[sopt] = v - except: pass - return 1 + # replacement found: + rplcmnt = 1 + try: # set it in map-vars (consider different python versions): + defaults[sopt] = v + except: + # try to set in first default map (corresponding vars): + try: + defaults._maps[0][sopt] = v + except: # pragma: no cover + # no way to update vars chain map - overwrite defaults: + self._defaults[sopt] = v + return rplcmnt @property def share_config(self): diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 577a5a16..2248ec34 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -351,7 +351,7 @@ class DefinitionInitConfigReader(ConfigReader): return self._defCache[optname] except KeyError: try: - v = self.get("Definition", optname, vars=self._pOpts) + v = self._cfg.get_ex("Definition", optname, vars=self._pOpts) except (NoSectionError, NoOptionError, ValueError): v = None self._defCache[optname] = v diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0472b770..6c0d9226 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -188,7 +188,7 @@ y = %(jail/y)s self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"') - self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"') self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"') self.assertRaises(Exception, self.c.get, 'test', 'x') @@ -437,9 +437,20 @@ class FilterReaderTest(unittest.TestCase): self.assertSortedEqual(c, output) def testFilterReaderSubstitionKnown(self): - output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=']] + output = [['set', 'jailname', 'addfailregex', '^to=test,sweet@example.com,test2,sweet@example.com fromip=$']] filterName, filterOpt = extractOptions( - 'substition[honeypot=",", sweet="test,,test2"]') + 'substition[failregex="^$", honeypot=",", sweet="test,,test2"]') + filterReader = FilterReader('substition', "jailname", filterOpt, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertSortedEqual(c, output) + + def testFilterReaderSubstitionSection(self): + output = [['set', 'jailname', 'addfailregex', '^\s*to=fail2ban@localhost fromip=\s*$']] + filterName, filterOpt = extractOptions( + 'substition[failregex="^\s*\s*$", honeypot=""]') filterReader = FilterReader('substition', "jailname", filterOpt, share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 8e175006..92e08bfc 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -318,7 +318,7 @@ def with_foreground_server_thread(startextra={}): Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) self.assertTrue(phase.get('start', None)) # wait for server (socket and ready): - self._wait_for_srv(tmp, True, startparams=startparams) + self._wait_for_srv(tmp, True, startparams=startparams, phase=phase) DefLogSys.info('=== within server: begin ===') self.pruneLog() # several commands to server in body of decorated function: @@ -368,12 +368,12 @@ class Fail2banClientServerBase(LogCaptureTestCase): else: raise FailExitException() - def _wait_for_srv(self, tmp, ready=True, startparams=None): + def _wait_for_srv(self, tmp, ready=True, startparams=None, phase={}): try: sock = pjoin(tmp, "f2b.sock") # wait for server (socket): - ret = Utils.wait_for(lambda: exists(sock), MAX_WAITTIME) - if not ret: + ret = Utils.wait_for(lambda: phase.get('end') or exists(sock), MAX_WAITTIME) + if not ret or phase.get('end'): # pragma: no cover - test-failure case only raise Exception( 'Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,) @@ -381,7 +381,7 @@ class Fail2banClientServerBase(LogCaptureTestCase): if ready: # wait for communication with worker ready: ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) - if not ret: + if not ret: # pragma: no cover - test-failure case only raise Exception( 'Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,) @@ -405,10 +405,12 @@ class Fail2banClientServerBase(LogCaptureTestCase): # start and wait to end (foreground): logSys.debug("start of test worker") phase['start'] = True - self.execCmd(SUCCESS, ("-f",) + startparams, "start") - # end : - phase['end'] = True - logSys.debug("end of test worker") + try: + self.execCmd(SUCCESS, ("-f",) + startparams, "start") + finally: + # end : + phase['end'] = True + logSys.debug("end of test worker") @with_foreground_server_thread() def testStartForeground(self, tmp, startparams): @@ -1173,7 +1175,7 @@ class Fail2banServerTest(Fail2banClientServerBase): @with_foreground_server_thread(startextra={ # create log-file (avoid "not found" errors): 'create_before_start': ('%(tmp)s/blck-failures.log',), - # we need action.d/nginx-block-map.conf: + # we need action.d/nginx-block-map.conf and blocklist_de: 'use_stock_cfg': ('action.d',), # jail-config: 'jails': ( @@ -1182,6 +1184,8 @@ class Fail2banServerTest(Fail2banClientServerBase): 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + ' blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; \', email="Fail2Ban ", ' + 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=]', 'filter =', 'datepattern = ^Epoch', 'failregex = ^ failure "[^"]+" - ', @@ -1219,6 +1223,14 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check blocklist_de substitution (e. g. new-line after ): + self.assertLogged( + "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " + " --data apikey=TEST-API-KEY --data service=nginx-blck-lst ", + "stdout: ' --data format=text --user-agent fail2ban-test-agent", + all=True, wait=MID_WAITTIME + ) + # unban 1, 2 and 5: self.execCmd(SUCCESS, startparams, 'unban', '125-000-001', '125-000-002', '125-000-005') _out_file(mpfn) diff --git a/fail2ban/tests/files/filter.d/substition.conf b/fail2ban/tests/files/filter.d/substition.conf index aaf62eae..862a3cac 100644 --- a/fail2ban/tests/files/filter.d/substition.conf +++ b/fail2ban/tests/files/filter.d/substition.conf @@ -1,3 +1,6 @@ +[DEFAULT] + +honeypot = fail2ban@localhost [Definition]