From 7f55be3fadc1701ab08e0c97936748e3eec98c12 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 15:31:28 +0200 Subject: [PATCH 1/4] amend to b6bb2f88c1dbb111647269590d80d95f72c81c3e: datepattern right word boundary - prevents confusions if end of date-pattern (e.g. optional year part) misleadingly match not date values (see gh-1507) test cases extended to check ambiguous "unbound" patterns in log lines (match/miss resp. positive/negative cases) --- fail2ban/server/datetemplate.py | 8 ++++--- fail2ban/tests/fail2banregextestcase.py | 30 ++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 8210dad4..0a5ec55a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -64,7 +64,7 @@ class DateTemplate(object): def getRegex(self): return self._regex - def setRegex(self, regex, wordBegin=True): + def setRegex(self, regex, wordBegin=True, wordEnd=True): """Sets regex to use for searching for date in log line. Parameters @@ -82,8 +82,10 @@ class DateTemplate(object): If regular expression fails to compile """ regex = regex.strip() - if (wordBegin and not re.search(r'^\^', regex)): - regex = r'\b' + regex + if wordBegin and not re.search(r'^\^', regex): + regex = r'(?=^|\b|\W)' + regex + if wordEnd and not re.search(r'\$$', regex): + regex += r'(?=\b|\W|$)' self._regex = regex self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 1119efdb..fb1bc6e3 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -187,4 +187,32 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') - + def testAmbiguousDatePattern(self): + for (matched, args) in ( + # positive case: + (1, ('Test failure Jan 23 21:59:59 for 192.0.2.1', r'for $')), + # ambiguous "unbound" patterns (missed): + (0, ('Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1', r'for $')), + (0, ('Test failure Jan 23 21:59:59123456789 for 192.0.2.1', r'for $')), + # ambiguous "no optional year" patterns (matched): + (1, ('Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), + (1, ('[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), + # direct specified patterns: + (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), + (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), + ): + logSys.debug('== test: %r', args) + (opts, args, fail2banRegex) = _Fail2banRegex(*args) + self.assertTrue(fail2banRegex.start(opts, args)) + matchedLog = 'Lines: 1 lines, 0 ignored, 1 matched, 0 missed' + missedLog = 'Lines: 1 lines, 0 ignored, 0 matched, 1 missed' + if matched: + self.assertLogged(matchedLog) + self.assertNotLogged(missedLog) + else: + self.assertNotLogged(matchedLog) + self.assertLogged(missedLog) + self.pruneLog() From 8e09be5fc8810a338c323922975410501fa927d7 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 18:53:35 +0200 Subject: [PATCH 2/4] test cases for boundaries for date-pattern extended (negative/positive, left/right) --- fail2ban/tests/fail2banregextestcase.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index fb1bc6e3..59ba60be 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -200,9 +200,18 @@ class Fail2banRegexTest(LogCaptureTestCase): # direct specified patterns: (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), + (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1', r'^Attempt\s+IP address $')), + (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]', r'^Attempt IP address , date: $')), + # direct specified patterns (begin/end, missed): + (0, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003', '^')), + (0, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1', '$')), + # direct specified patterns (begin/end, matched): + (1, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003', '^')), + (1, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1', '$')), ): logSys.debug('== test: %r', args) (opts, args, fail2banRegex) = _Fail2banRegex(*args) From 0bdee2556f72da7fbf5a14e8abcb402607966811 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 19:35:11 +0200 Subject: [PATCH 3/4] testAmbiguousDatePattern rewritten with DateDetector/DatePatternRegex directly (moved to misctestcase.py) --- fail2ban/tests/fail2banregextestcase.py | 39 +-------------------- fail2ban/tests/misctestcase.py | 45 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 59ba60be..1119efdb 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -187,41 +187,4 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') - def testAmbiguousDatePattern(self): - for (matched, args) in ( - # positive case: - (1, ('Test failure Jan 23 21:59:59 for 192.0.2.1', r'for $')), - # ambiguous "unbound" patterns (missed): - (0, ('Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1', r'for $')), - (0, ('Test failure Jan 23 21:59:59123456789 for 192.0.2.1', r'for $')), - # ambiguous "no optional year" patterns (matched): - (1, ('Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), - (1, ('[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), - # direct specified patterns: - (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), - (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), - (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), - (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1', r'^Attempt\s+IP address $')), - (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]', r'^Attempt IP address , date: $')), - # direct specified patterns (begin/end, missed): - (0, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003', '^')), - (0, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1', '$')), - # direct specified patterns (begin/end, matched): - (1, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003', '^')), - (1, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1', '$')), - ): - logSys.debug('== test: %r', args) - (opts, args, fail2banRegex) = _Fail2banRegex(*args) - self.assertTrue(fail2banRegex.start(opts, args)) - matchedLog = 'Lines: 1 lines, 0 ignored, 1 matched, 0 missed' - missedLog = 'Lines: 1 lines, 0 ignored, 0 matched, 1 missed' - if matched: - self.assertLogged(matchedLog) - self.assertNotLogged(missedLog) - else: - self.assertNotLogged(matchedLog) - self.assertLogged(missedLog) - self.pruneLog() + diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index e2b4c0b6..4dfff2d4 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -37,6 +37,7 @@ from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger from ..helpers import splitwords +from ..server.datedetector import DateDetector from ..server.datetemplate import DatePatternRegex @@ -340,3 +341,47 @@ class CustomDateFormatsTest(unittest.TestCase): self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) + + def testAmbiguousDatePattern(self): + defDD = DateDetector() + defDD.addDefaultTemplate() + logSys = DefLogSys + for (matched, dp, line) in ( + # positive case: + ('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'), + # ambiguous "unbound" patterns (missed): + (False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'), + (False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'), + # ambiguous "no optional year" patterns (matched): + ('Aug 8 11:25:50', None, 'Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 14430f2329b8 Authentication failed from 192.0.2.1'), + # direct specified patterns: + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'), + # direct specified patterns (begin/end, missed): + (False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'), + (False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'), + # direct specified patterns (begin/end, matched): + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'), + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'), + ): + logSys.debug('== test: %r', (matched, dp, line)) + if dp is None: + dd = defDD + else: + dp = DatePatternRegex(dp) + dd = DateDetector() + dd.appendTemplate(dp) + date = dd.getTime(line) + if matched: + self.assertTrue(date) + self.assertEqual(matched, date[1].group()) + else: + self.assertEqual(date, None) From 9935cf19c1b8aa9779e419b3f2c498531b362f1c Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 19:54:11 +0200 Subject: [PATCH 4/4] description provided, ChangeLog entries added --- ChangeLog | 3 +++ fail2ban/server/datetemplate.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ecb8cc87..c4444e2a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,7 @@ releases. uses "fail2ban-python" now); * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); +* Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); * `filter.d/ignorecommands/apache-fakegooglebot` - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) @@ -33,6 +34,8 @@ releases. ### New Features ### Enhancements +* DateTemplate regexp extended with the word-end boundary, additionally to + word-start boundary * Introduces new command "fail2ban-python", as automatically created symlink to python executable, where fail2ban currently installed (resp. its modules are located): - allows to use the same version, fail2ban currently running, e.g. in diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 0a5ec55a..8e602289 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -72,8 +72,12 @@ class DateTemplate(object): regex : str The regex the template will use for searching for a date. wordBegin : bool - Defines whether the regex should be modified to search at - beginning of a word, by adding "\\b" to start of regex. + Defines whether the regex should be modified to search at beginning of a + word, by adding special boundary r'(?=^|\b|\W)' to start of regex. + Default True. + wordEnd : bool + Defines whether the regex should be modified to search at end of a word, + by adding special boundary r'(?=\b|\W|$)' to end of regex. Default True. Raises