From f547a7c7b1f9b6e72ad8fae6ec5e18a6bc7bfbcc Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 26 Jan 2018 20:25:11 +0100 Subject: [PATCH 1/9] LogCaptureTestCase: use almost non-blocking handling by getvalue/_is_logged (especially important in tests with waiting for logged via `assertLogged(..., wait=TO)`): - try to acquire lock without blocking, if not possible - return cached/empty (max 5 times, otherwise do lock); - minimized time of the lock of messages list; - avoid sporadic dead-locking during cross lock together with lock within handling of self._strm. --- fail2ban/tests/utils.py | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 2bcc587b..f681db76 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -640,7 +640,9 @@ class LogCaptureTestCase(unittest.TestCase): def __init__(self, lazy=True): self._lock = threading.Lock() self._val = None + self._dirty = True self._recs = list() + self._nolckCntr = 0 self._strm = StringIO() logging.Handler.__init__(self) if lazy: @@ -650,10 +652,11 @@ class LogCaptureTestCase(unittest.TestCase): """Truncate the internal buffer and records.""" if size: raise Exception('invalid size argument: %r, should be None or 0' % size) + self._val = None + self._dirty = True with self._lock: - self._strm.truncate(0) - self._val = None self._recs = list() + self._strm.truncate(0) def __write(self, record): msg = record.getMessage() + '\n' @@ -664,29 +667,42 @@ class LogCaptureTestCase(unittest.TestCase): def getvalue(self): """Return current buffer as whole string.""" - with self._lock: - # cached: - if self._val is not None: - return self._val - # submit already emitted (delivered to handle) records: - for record in self._recs: - self.__write(record) - self._recs = list() - # cache and return: - self._val = self._strm.getvalue() + # if cached (still unchanged/no write operation), we don't need to enter lock: + if not self._dirty: return self._val + # try to lock, if not possible - return cached/empty (max 5 times): + lck = self._lock.acquire(False) + if not lck: # pargma: no cover (may be too sporadic on slow systems) + self._nolckCntr += 1 + if self._nolckCntr <= 5: + return self._val if self._val is not None else '' + self._nolckCntr = 0 + self._lock.acquire() + # minimize time of lock, avoid dead-locking during cross lock within self._strm ... + try: + recs = self._recs + self._recs = list() + finally: + self._lock.release() + # submit already emitted (delivered to handle) records: + for record in recs: + self.__write(record) + # cache and return: + self._val = self._strm.getvalue() + self._dirty = False + return self._val def handle(self, record): # pragma: no cover """Handle the specified record direct (not lazy)""" - with self._lock: - self._val = None - self.__write(record) + self.__write(record) + self._dirty = True def _handle_lazy(self, record): """Lazy handle the specified record on demand""" with self._lock: - self._val = None self._recs.append(record) + # logged - causes changed string buffer (signal by set _dirty): + self._dirty = True def setUp(self): From 435f359a065576f2537c07d24fc1d58277cd190e Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 26 Jan 2018 21:34:10 +0100 Subject: [PATCH 2/9] allow substitute section-related parameters like `` in all config-readers as well as during substitute after supply of init arguments; test cases extended; --- fail2ban/client/configparserinc.py | 76 +++++++++++++++---- fail2ban/client/configreader.py | 2 +- fail2ban/tests/clientreadertestcase.py | 17 ++++- fail2ban/tests/files/filter.d/substition.conf | 3 + 4 files changed, 80 insertions(+), 18 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 722f4618..70dfd91b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -33,7 +33,7 @@ if sys.version_info >= (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ - InterpolationMissingOptionError, NoSectionError + InterpolationMissingOptionError, NoOptionError, NoSectionError # And interpolation of __name__ was simply removed, thus we need to # decorate default interpolator to handle it @@ -63,7 +63,7 @@ if sys.version_info >= (3,2): else: # pragma: no cover from ConfigParser import SafeConfigParser, \ - InterpolationMissingOptionError, NoSectionError + InterpolationMissingOptionError, NoOptionError, NoSectionError # Interpolate missing known/option as option from default section SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some @@ -112,6 +112,8 @@ after = 1.conf SECTION_NAME = "INCLUDES" + SECTION_OPTNAME_CRE = re.compile(r'^([\w\-]+)/([^\s>]+)$') + SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s') CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$") @@ -131,7 +133,36 @@ after = 1.conf SafeConfigParser.__init__(self, *args, **kwargs) self._cfg_share = share_config - def _map_section_options(self, section, option, rest, map): + def get_ex(self, section, option, raw=False, vars={}): + """Get an option value for a given section. + + In opposite to `get`, it differentiate session-related option name like `sec/opt`. + """ + sopt = None + # if option name contains section: + if '/' in option: + sopt = SafeConfigParserWithIncludes.SECTION_OPTNAME_CRE.search(option) + # try get value from named section/option: + if sopt: + sec = sopt.group(1) + opt = sopt.group(2) + seclwr = sec.lower() + if seclwr == 'known': + # try get value firstly from known options, hereafter from current section: + sopt = ('KNOWN/'+section, section) + else: + sopt = (sec,) if seclwr != 'default' else ("DEFAULT",) + for sec in sopt: + try: + v = self.get(sec, opt, raw=raw) + return v + except (NoSectionError, NoOptionError) as e: + pass + # get value of section/option using given section and vars (fallback): + v = self.get(section, option, raw=raw, vars=vars) + return v + + def _map_section_options(self, section, option, rest, defaults): """ Interpolates values of the section options (name syntax `%(section/option)s`). @@ -139,37 +170,54 @@ after = 1.conf """ if '/' not in rest or '%(' not in rest: # pragma: no cover return 0 + rplcmnt = 0 soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest) if not soptrep: # pragma: no cover return 0 for sopt, opt in soptrep: - if sopt not in map: + if sopt not in defaults: sec = sopt[:~len(opt)] seclwr = sec.lower() if seclwr != 'default': + usedef = 0 if seclwr == 'known': # try get raw value from known options: try: v = self._sections['KNOWN/'+section][opt] except KeyError: # fallback to default: - try: - v = self._defaults[opt] - except KeyError: # pragma: no cover - continue + usedef = 1 else: # get raw value of opt in section: - v = self.get(sec, opt, raw=True) + try: + # if section not found - ignore: + try: + sec = self._sections[sec] + except KeyError: # pragma: no cover + continue + v = sec[opt] + except KeyError: # pragma: no cover + # fallback to default: + usedef = 1 else: + usedef = 1 + if usedef: try: v = self._defaults[opt] except KeyError: # pragma: no cover continue - self._defaults[sopt] = v - try: # for some python versions need to duplicate it in map-vars also: - map[sopt] = v - except: pass - return 1 + # replacement found: + rplcmnt = 1 + try: # set it in map-vars (consider different python versions): + defaults[sopt] = v + except: + # try to set in first default map (corresponding vars): + try: + defaults._maps[0][sopt] = v + except: # pragma: no cover + # no way to update vars chain map - overwrite defaults: + self._defaults[sopt] = v + return rplcmnt @property def share_config(self): diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 577a5a16..2248ec34 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -351,7 +351,7 @@ class DefinitionInitConfigReader(ConfigReader): return self._defCache[optname] except KeyError: try: - v = self.get("Definition", optname, vars=self._pOpts) + v = self._cfg.get_ex("Definition", optname, vars=self._pOpts) except (NoSectionError, NoOptionError, ValueError): v = None self._defCache[optname] = v diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0472b770..6c0d9226 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -188,7 +188,7 @@ y = %(jail/y)s self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"') self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"') - self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') + self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"') self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"') self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"') self.assertRaises(Exception, self.c.get, 'test', 'x') @@ -437,9 +437,20 @@ class FilterReaderTest(unittest.TestCase): self.assertSortedEqual(c, output) def testFilterReaderSubstitionKnown(self): - output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=']] + output = [['set', 'jailname', 'addfailregex', '^to=test,sweet@example.com,test2,sweet@example.com fromip=$']] filterName, filterOpt = extractOptions( - 'substition[honeypot=",", sweet="test,,test2"]') + 'substition[failregex="^$", honeypot=",", sweet="test,,test2"]') + filterReader = FilterReader('substition', "jailname", filterOpt, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertSortedEqual(c, output) + + def testFilterReaderSubstitionSection(self): + output = [['set', 'jailname', 'addfailregex', '^\s*to=fail2ban@localhost fromip=\s*$']] + filterName, filterOpt = extractOptions( + 'substition[failregex="^\s*\s*$", honeypot=""]') filterReader = FilterReader('substition', "jailname", filterOpt, share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) filterReader.read() diff --git a/fail2ban/tests/files/filter.d/substition.conf b/fail2ban/tests/files/filter.d/substition.conf index aaf62eae..862a3cac 100644 --- a/fail2ban/tests/files/filter.d/substition.conf +++ b/fail2ban/tests/files/filter.d/substition.conf @@ -1,3 +1,6 @@ +[DEFAULT] + +honeypot = fail2ban@localhost [Definition] From 03b577d7b92a120e325abe20a99b6956a7e0657c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 12:27:03 +0100 Subject: [PATCH 3/9] action.d/blocklist_de.conf: fixed tag substitution (in 0.10 it can be variables supplied via shell-arguments), expand `` with trailing newline; tests extended; closes gh-2028 --- config/action.d/blocklist_de.conf | 4 +--- fail2ban/tests/fail2banclienttestcase.py | 30 +++++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 2f31d8b9..246f90f7 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" +actionban = lgm=$(printf 'logs=%%s\n...' ""); curl --fail --data-urlencode "server=" --data "apikey=" --data "service=" --data "ip=" --data-urlencode "$lgm" --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -64,8 +64,6 @@ actionban = curl --fail --data-urlencode 'server=' --data 'apikey=\', email="Fail2Ban ", ' + 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=]', 'filter =', 'datepattern = ^Epoch', 'failregex = ^ failure "[^"]+" - ', @@ -1219,6 +1223,14 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check blocklist_de substitution: + self.assertLogged( + "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " + " --data apikey=TEST-API-KEY --data service=nginx-blck-lst ", + "stdout: '... --data format=text --user-agent fail2ban-test-agent", + all=True, wait=MID_WAITTIME + ) + # unban 1, 2 and 5: self.execCmd(SUCCESS, startparams, 'unban', '125-000-001', '125-000-002', '125-000-005') _out_file(mpfn) From 0be0e43d475ab9cec83b747134023279d1c95eb3 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 12:52:26 +0100 Subject: [PATCH 4/9] amend to 03b577d7b92a120e325abe20a99b6956a7e0657c: add new-line after matches via tag `
` without usage of interim variable --- config/action.d/blocklist_de.conf | 2 +- fail2ban/tests/fail2banclienttestcase.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 246f90f7..b9cd0584 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = lgm=$(printf 'logs=%%s\n...' ""); curl --fail --data-urlencode "server=" --data "apikey=" --data "service=" --data "ip=" --data-urlencode "$lgm" --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" +actionban = curl --fail --data-urlencode "server=" --data "apikey=" --data "service=" --data "ip=" --data-urlencode "logs=
" --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 92fdfa5c..e346f09d 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1223,11 +1223,11 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) - # check blocklist_de substitution: + # check blocklist_de substitution (e. g. new-line after ): self.assertLogged( "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " - " --data apikey=TEST-API-KEY --data service=nginx-blck-lst ", - "stdout: '... --data format=text --user-agent fail2ban-test-agent", + " --data apikey=TEST-API-KEY --data service=nginx-blck-lst ", + "stdout: ' --data format=text --user-agent fail2ban-test-agent", all=True, wait=MID_WAITTIME ) From 0ed11817c197f045ef7a5f82cc9fb33a4d5f1657 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 13:30:31 +0100 Subject: [PATCH 5/9] restore coverage: no cover for normally unreachable scopes (only if test cases failed) --- fail2ban/tests/fail2banclienttestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index e346f09d..92e08bfc 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -373,7 +373,7 @@ class Fail2banClientServerBase(LogCaptureTestCase): sock = pjoin(tmp, "f2b.sock") # wait for server (socket): ret = Utils.wait_for(lambda: phase.get('end') or exists(sock), MAX_WAITTIME) - if not ret or phase.get('end'): + if not ret or phase.get('end'): # pragma: no cover - test-failure case only raise Exception( 'Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,) @@ -381,7 +381,7 @@ class Fail2banClientServerBase(LogCaptureTestCase): if ready: # wait for communication with worker ready: ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) - if not ret: + if not ret: # pragma: no cover - test-failure case only raise Exception( 'Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,) From 442b0b1c59c904fb6961a605be399f57029cabd5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 14:41:38 +0100 Subject: [PATCH 6/9] extends date-detector with long epoch (LEPOCH) to parse milliseconds/microseconds posix-dates; provide opportunity to specify own regex-pattern to match epoch date-time, e. g. "^\[{EPOCH}\]"; closes gh-2029 --- fail2ban/server/datedetector.py | 23 +++++++++----- fail2ban/server/datetemplate.py | 27 ++++++++++++++--- fail2ban/tests/datedetectortestcase.py | 42 ++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 42df308e..13d70699 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -26,7 +26,8 @@ import time from threading import Lock -from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch +from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch, \ + RE_EPOCH_PATTERN from .strptime import validateTimeZone from .utils import Utils from ..helpers import getLogger @@ -36,7 +37,7 @@ logSys = getLogger(__name__) logLevel = 6 -RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE) +RE_DATE_PREMATCH = re.compile(r"(?(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" + self._longFrm = longFrm; + epochRE = r"\d{10,11}\b(?:\.\d{3,6})?" + if longFrm: + self.name = "LongEpoch"; + epochRE = r"\d{10,11}(?:\d{3}(?:\d{3})?)?" + if pattern: + regex = RE_EPOCH_PATTERN.sub("(%s)" % epochRE, pattern) + self.setRegex(regex) + elif not lineBeginOnly: + regex = r"((?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))%s)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" % epochRE self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored else: - regex = r"((?P(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))" + regex = r"((?P(?<=^\[))?%s)(?(square)(?=\]))" % epochRE self.setRegex(regex, wordBegin='start', wordEnd=True) def getDate(self, line, dateMatch=None, default_tz=None): @@ -220,8 +231,14 @@ class DateEpoch(DateTemplate): if not dateMatch: dateMatch = self.matchDate(line) if dateMatch: + v = dateMatch.group(1) # extract part of format which represents seconds since epoch - return (float(dateMatch.group(1)), dateMatch) + if self._longFrm and len(v) >= 13: + if len(v) >= 16: + v = float(v) / 1000000 + else: + v = float(v) / 1000 + return (float(v), dateMatch) class DatePatternRegex(DateTemplate): diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 02facf30..69473c9d 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -77,6 +77,48 @@ class DateDetectorTest(LogCaptureTestCase): log = date + " [sshd] error: PAM: Authentication failure" datelog = self.datedetector.getTime(log) self.assertFalse(datelog) + + def testGetEpochMsTime(self): + self.__datedetector = DateDetector() + self.__datedetector.appendTemplate('LEPOCH') + # correct short/long epoch time, using all variants: + for fact in (1, 1000, 1000000): + for dateUnix in (1138049999, 32535244799): + for date in ("%s", "[%s]", "[%s]", "audit(%s:101)"): + dateLong = dateUnix * fact + date = date % dateLong + log = date + " [sshd] error: PAM: Authentication failure" + datelog = self.datedetector.getTime(log) + self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) + ( datelog, matchlog ) = datelog + self.assertEqual(int(datelog), dateUnix) + self.assertEqual(matchlog.group(1), str(dateLong)) + # wrong, no epoch time (< 10 digits, more as 17 digits, begin/end of word) : + for dateUnix in ('123456789', '999999999999999999', '1138049999A', 'A1138049999'): + for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): + date = date % dateUnix + log = date + " [sshd] error: PAM: Authentication failure" + datelog = self.datedetector.getTime(log) + self.assertFalse(datelog) + + def testGetEpochPattern(self): + self.__datedetector = DateDetector() + self.__datedetector.appendTemplate('\|\s{LEPOCH}(?=\s\|)') + # correct short/long epoch time, using all variants: + for fact in (1, 1000, 1000000): + for dateUnix in (1138049999, 32535244799): + dateLong = dateUnix * fact + log = "auth-error | %s | invalid password" % dateLong + datelog = self.datedetector.getTime(log) + self.assertTrue(datelog, "Parse epoch time failed: %r" % (log,)) + ( datelog, matchlog ) = datelog + self.assertEqual(int(datelog), dateUnix) + self.assertEqual(matchlog.group(1), str(dateLong)) + # wrong epoch time format (does not match pattern): + for log in ("test%s123", "test-right | %stest", "test%s | test-left"): + log = log % dateLong + datelog = self.datedetector.getTime(log) + self.assertFalse(datelog) def testGetTime(self): log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure" From 3e8098d4274b5f00585c201b57bc7165f02c3218 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 15:10:17 +0100 Subject: [PATCH 7/9] python 3.x compatibility: fix replacement string (may fail with errors like `bad escape \d ...`, etc) --- fail2ban/server/datetemplate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 4c9bb4e3..cd4592c7 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -204,7 +204,7 @@ class DateEpoch(DateTemplate): self.name = "LongEpoch"; epochRE = r"\d{10,11}(?:\d{3}(?:\d{3})?)?" if pattern: - regex = RE_EPOCH_PATTERN.sub("(%s)" % epochRE, pattern) + regex = RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) self.setRegex(regex) elif not lineBeginOnly: regex = r"((?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))%s)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" % epochRE From dcbf9048760d3ada651789a0bb71b18e998ec040 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 30 Jan 2018 16:40:04 +0100 Subject: [PATCH 8/9] allow to parse milliseconds as float + more test cases; normalize capturing with epoch-pattern match - similar to `{DATE}` should capture and cut out the whole pattern match from the log-line; --- fail2ban/server/datetemplate.py | 11 +++++++---- fail2ban/tests/datedetectortestcase.py | 2 +- fail2ban/tests/fail2banregextestcase.py | 11 +++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index cd4592c7..49fa0c66 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -199,12 +199,15 @@ class DateEpoch(DateTemplate): DateTemplate.__init__(self) self.name = "Epoch" self._longFrm = longFrm; + self._grpIdx = 1 epochRE = r"\d{10,11}\b(?:\.\d{3,6})?" if longFrm: self.name = "LongEpoch"; - epochRE = r"\d{10,11}(?:\d{3}(?:\d{3})?)?" + epochRE = r"\d{10,11}(?:\d{3}(?:\.\d{1,6}|\d{3})?)?" if pattern: - regex = RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) + # pattern should capture/cut out the whole match: + regex = "(" + RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) + ")" + self._grpIdx = 2 self.setRegex(regex) elif not lineBeginOnly: regex = r"((?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))%s)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" % epochRE @@ -231,10 +234,10 @@ class DateEpoch(DateTemplate): if not dateMatch: dateMatch = self.matchDate(line) if dateMatch: - v = dateMatch.group(1) + v = dateMatch.group(self._grpIdx) # extract part of format which represents seconds since epoch if self._longFrm and len(v) >= 13: - if len(v) >= 16: + if len(v) >= 16 and '.' not in v: v = float(v) / 1000000 else: v = float(v) / 1000 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 69473c9d..36471489 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -103,7 +103,7 @@ class DateDetectorTest(LogCaptureTestCase): def testGetEpochPattern(self): self.__datedetector = DateDetector() - self.__datedetector.appendTemplate('\|\s{LEPOCH}(?=\s\|)') + self.__datedetector.appendTemplate('(?<=\|\s){LEPOCH}(?=\s\|)') # correct short/long epoch time, using all variants: for fact in (1, 1000, 1000000): for dateUnix in (1138049999, 32535244799): diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index f3a51773..148d774c 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -290,6 +290,17 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + def testRegexEpochPatterns(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "-r", "-d", "^\[{LEPOCH}\]\s+", "--maxlines", "5", + "[1516469849] 192.0.2.1 FAIL: failure\n" + "[1516469849551] 192.0.2.2 FAIL: failure\n" + "[1516469849551000] 192.0.2.3 FAIL: failure\n" + "[1516469849551.000] 192.0.2.4 FAIL: failure", + r"^ FAIL\b" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 4 matched, 0 missed') def testWrongFilterFile(self): # use test log as filter file to cover eror cases... From 3a1c38695843e37493e88ea7354eccda51a0abcf Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 31 Jan 2018 12:18:56 +0100 Subject: [PATCH 9/9] Update ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3813a156..e0e244df 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,10 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition ### New Features ### Enhancements +* date-detector extended with long epoch (`LEPOCH`) to parse milliseconds/microseconds posix-dates (gh-2029); +* possibility to specify own regex-pattern to match epoch date-time, e. g. `^\[{EPOCH}\]` or `^\[{LEPOCH}\]` (gh-2038); + the epoch-pattern similar to `{DATE}` patterns does the capture and cuts out the match of whole pattern from the log-line, + e. g. date-pattern `^\[{LEPOCH}\]\s+:` will match and cut out `[1516469849551000] :` from begin of the log-line. ver. 0.10.2 (2018/01/18) - nothing-burns-like-the-cold