From e8f217390449120a3c3c7fe4ca502b37eb8d2c56 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Tue, 23 May 2017 17:27:12 +0200 Subject: [PATCH] New logtimezone jail option This new option allows to force the time zone on log lines that don't bear a time zone indication (GitHub issue #1773), so it behaves actually with respect to log line contents as a default time zone. For the time being, only fixed offset timezones (UTC or UTC[+-]hhmm) are supported, but the implementation is designed to later on treat the case of logical timezones with DST, e.g., Europe/Paris etc. In particular, the timezone name gets passed all the way to the strptime module, and the resulting offset is computed for the given log line, even though for now, it doesn't actually depend on it. Also, the DateTemplate subclass gets to choose whether to use it or not. For instance, it doesn't make sense to apply a time zone offset to Unix timestamps. The drawback is to introduce an API change for DateTemplate. I hope it's internal enough for that not being a problem. --- fail2ban/client/jailreader.py | 1 + fail2ban/server/datedetector.py | 4 +-- fail2ban/server/datetemplate.py | 16 +++++++---- fail2ban/server/filter.py | 27 ++++++++++++++++-- fail2ban/server/server.py | 6 ++++ fail2ban/server/strptime.py | 39 ++++++++++++++++++++++++-- fail2ban/server/transmitter.py | 6 ++++ fail2ban/tests/clientreadertestcase.py | 12 ++++++++ fail2ban/tests/config/jail.conf | 4 +++ fail2ban/tests/datedetectortestcase.py | 15 ++++++++++ fail2ban/tests/filtertestcase.py | 10 +++++++ fail2ban/tests/servertestcase.py | 4 +++ 12 files changed, 133 insertions(+), 11 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index ca092990..ce0ed3b6 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -101,6 +101,7 @@ class JailReader(ConfigReader): ["string", "filter", ""]] opts = [["bool", "enabled", False], ["string", "logpath", None], + ["string", "logtimezone", None], ["string", "logencoding", None], ["string", "backend", "auto"], ["int", "maxretry", None], diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index dd5d198f..39a15828 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -423,7 +423,7 @@ class DateDetector(object): logSys.log(logLevel, " no template.") return (None, None) - def getTime(self, line, timeMatch=None): + def getTime(self, line, timeMatch=None, default_tz=None): """Attempts to return the date on a log line using templates. This uses the templates' `getDate` method in an attempt to find @@ -449,7 +449,7 @@ class DateDetector(object): template = timeMatch[1] if template is not None: try: - date = template.getDate(line, timeMatch[0]) + date = template.getDate(line, timeMatch[0], default_tz=default_tz) if date is not None: if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug logSys.log(logLevel, " got time %f for %r using template %s", diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 1d0b014b..44ea54a8 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -158,7 +158,7 @@ class DateTemplate(object): return dateMatch @abstractmethod - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Abstract method, which should return the date for a log line This should return the date for a log line, typically taking the @@ -169,6 +169,8 @@ class DateTemplate(object): ---------- line : str Log line, of which the date should be extracted from. + default_tz: if no explicit time zone is present in the line + passing this will interpret it as in that time zone. Raises ------ @@ -200,13 +202,14 @@ class DateEpoch(DateTemplate): regex = r"((?P(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))" self.setRegex(regex, wordBegin='start', wordEnd=True) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. Parameters ---------- line : str Log line, of which the date should be extracted from. + default_tz: ignored, Unix timestamps are time zone independent Returns ------- @@ -277,7 +280,7 @@ class DatePatternRegex(DateTemplate): regex = r'(?iu)' + regex super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. This uses a custom version of strptime, using the named groups @@ -287,6 +290,7 @@ class DatePatternRegex(DateTemplate): ---------- line : str Log line, of which the date should be extracted from. + default_tz: optionally used to correct timezone Returns ------- @@ -297,7 +301,8 @@ class DatePatternRegex(DateTemplate): if not dateMatch: dateMatch = self.matchDate(line) if dateMatch: - return reGroupDictStrptime(dateMatch.groupdict()), dateMatch + return (reGroupDictStrptime(dateMatch.groupdict(), default_tz=default_tz), + dateMatch) class DateTai64n(DateTemplate): @@ -315,13 +320,14 @@ class DateTai64n(DateTemplate): # We already know the format for TAI64N self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. Parameters ---------- line : str Log line, of which the date should be extracted from. + default_tz: ignored, since TAI is time zone independent Returns ------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index fce02a7a..8d1eb856 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -40,6 +40,7 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from .utils import Utils from ..helpers import getLogger, PREFER_ENC +from .strptime import validateTimeZone # Gets the instance of the logger. logSys = getLogger(__name__) @@ -102,6 +103,8 @@ class Filter(JailThread): self.checkAllRegex = False ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## if set, treat log lines without explicit time zone to be in this time zone + self.logtimezone = None ## Ticks counter self.ticks = 0 @@ -307,6 +310,22 @@ class Filter(JailThread): return pattern, templates[0].name return None + ## + # Set the log default time zone + # + # @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM) + + def setLogTimeZone(self, tz): + self.logtimezone = validateTimeZone(tz) + + ## + # Get the log default timezone + # + # @return symbolic timezone (a string) + + def getLogTimeZone(self): + return self.logtimezone + ## # Set the maximum retry value. # @@ -621,7 +640,8 @@ class Filter(JailThread): self.__lastDate = date elif timeText: - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) + dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3], + default_tz=self.logtimezone) if dateTimeMatch is None: logSys.error("findFailure failed to parse timeText: %s", timeText) @@ -972,7 +992,10 @@ class FileFilter(Filter): break (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: - dateTimeMatch = self.dateDetector.getTime(line[timeMatch.start():timeMatch.end()], (timeMatch, template)) + dateTimeMatch = self.dateDetector.getTime( + line[timeMatch.start():timeMatch.end()], + (timeMatch, template), + default_tz=self.logtimezone) else: nextp = container.tell() if nextp > maxp: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 56367ed7..e3b22c44 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -379,6 +379,12 @@ class Server: def getDatePattern(self, name): return self.__jails[name].filter.getDatePattern() + def setLogTimeZone(self, name, tz): + self.__jails[name].filter.setLogTimeZone(tz) + + def getLogTimeZone(self, name): + return self.__jails[name].filter.getLogTimeZone() + def setIgnoreCommand(self, name, value): self.__jails[name].filter.setIgnoreCommand(value) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 55bdcc8c..aff9db92 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -17,6 +17,7 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +import re import time import calendar import datetime @@ -26,6 +27,7 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() +FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$') def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): """ Build century regex for last year and the next years (distance). @@ -78,7 +80,36 @@ def getTimePatternRE(): names[key] = "%%%s" % key return (patt, names) -def reGroupDictStrptime(found_dict, msec=False): + +def validateTimeZone(tz): + """Validate a timezone. + + For now this accepts only the UTC[+-]hhmm format. + In the future, it may be extended for named time zones (such as Europe/Paris) + present on the system, if a suitable tz library is present. + """ + m = FIXED_OFFSET_TZ_RE.match(tz) + if m is None: + raise ValueError("Unknown or unsupported time zone: %r" % tz) + return tz + +def zone2offset(tz, dt): + """Return the proper offset, in minutes according to given timezone at a given time. + + Parameters + ---------- + tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have + been validated already) + dt: datetime instance for offset computation + """ + if tz == 'UTC': + return 0 + unsigned = int(tz[4:6])*60 + int(tz[6:]) + if tz[3] == '-': + return -unsigned + return unsigned + +def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """Return time from dictionary of strptime fields This is tweaked from python built-in _strptime. @@ -88,7 +119,8 @@ def reGroupDictStrptime(found_dict, msec=False): found_dict : dict Dictionary where keys represent the strptime fields, and values the respective value. - + default_tz : default timezone to apply if nothing relevant is in found_dict + (may be a non-fixed one in the future) Returns ------- float @@ -209,6 +241,9 @@ def reGroupDictStrptime(found_dict, msec=False): # Actully create date date_result = datetime.datetime( year, month, day, hour, minute, second, fraction) + # Correct timezone if not supplied in the log linge + if tzoffset is None and default_tz is not None: + tzoffset = zone2offset(default_tz, date_result) # Add timezone info if tzoffset is not None: date_result -= datetime.timedelta(seconds=tzoffset * 60) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index bc9edd43..ecc2a138 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -261,6 +261,10 @@ class Transmitter: value = command[2] self.__server.setDatePattern(name, value) return self.__server.getDatePattern(name) + elif command[1] == "logtimezone": + value = command[2] + self.__server.setLogTimeZone(name, value) + return self.__server.getLogTimeZone(name) elif command[1] == "maxretry": value = command[2] self.__server.setMaxRetry(name, int(value)) @@ -363,6 +367,8 @@ class Transmitter: return self.__server.getFindTime(name) elif command[1] == "datepattern": return self.__server.getDatePattern(name) + elif command[1] == "logtimezone": + return self.__server.getLogTimeZone(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) elif command[1] == "maxlines": diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index bfa68e03..6cc2f659 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -196,6 +196,14 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.isEnabled()) self.assertLogged("Invalid action definition 'joho[foo'") + def testJailLogTimeZone(self): + jail = JailReader('tz_correct', basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + self.assertEqual(jail.options['logtimezone'], 'UTC+0200') + def testJailFilterBrokenDef(self): jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) @@ -533,10 +541,14 @@ class JailsReaderTest(LogCaptureTestCase): ]], ['add', 'parse_to_end_of_jail.conf', 'auto'], ['set', 'parse_to_end_of_jail.conf', 'addfailregex', ''], + ['set', 'tz_correct', 'addfailregex', ''], + ['set', 'tz_correct', 'logtimezone', 'UTC+0200'], ['start', 'emptyaction'], ['start', 'missinglogfiles'], ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'], + ['add', 'tz_correct', 'auto'], + ['start', 'tz_correct'], ['config-error', "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], ['config-error', diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 64c1b830..3dcbf634 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -47,3 +47,7 @@ action = thefunkychickendance [parse_to_end_of_jail.conf] enabled = true action = + +[tz_correct] +enabled = true +logtimezone = UTC+0200 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 39ab7173..16515f1b 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -89,6 +89,21 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(datelog, dateUnix) self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') + def testDefaultTimeZone(self): + log = "2017-01-23 15:00:00" + datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300') + # so in UTC, it was noon + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 12, 0, 0)) + + datelog, _ = self.datedetector.getTime(log, default_tz='UTC') + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 15, 0, 0)) + + datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430') + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 19, 30, 0)) + def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand """ diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 745dcca7..cb0edb06 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -289,6 +289,16 @@ class BasicFilter(unittest.TestCase): ("^%Y-%m-%d-%H%M%S.%f %z **", "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **")) + def testGetSetLogTimeZone(self): + self.assertEqual(self.filter.getLogTimeZone(), None) + self.filter.setLogTimeZone('UTC') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC') + self.filter.setLogTimeZone('UTC-0400') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC-0400') + self.filter.setLogTimeZone('UTC+0200') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC+0200') + self.assertRaises(ValueError, self.filter.setLogTimeZone, 'not-a-time-zone') + def testAssertWrongTime(self): self.assertRaises(AssertionError, lambda: _assert_equal_entries(self, diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 32f794a6..68b9951c 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -311,6 +311,10 @@ class Transmitter(TransmitterBase): "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) + def testLogTimeZone(self): + self.setGetTest("logtimezone", "UTC+0400", "UTC+0400", jail=self.jailName) + self.setGetTestNOK("logtimezone", "not-a-time-zone", jail=self.jailName) + def testJailUseDNS(self): self.setGetTest("usedns", "yes", jail=self.jailName) self.setGetTest("usedns", "warn", jail=self.jailName)