From ae7297e16b0188953bcdd18967bd22982b7fbfbb Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 Oct 2016 14:57:45 +0200 Subject: [PATCH] more precise date template handling (WARNING: this commit creates possible incompatibilities): - datedetector rewritten more strict as earlier; - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`); - more as one date pattern can be specified using option `datepattern` now (new-line separated); - some default options like `datepattern` can be specified directly in section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]` section, because of performance (each extra section costs time); - option `datepattern` can be specified in jail also (jails without filters); - if first group specified, only this will be cut out from search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match pattern, and leaves `date:[] failure ip...` for searching in filter); - faster match and fewer searching of appropriate templates (DateDetector.matchTime calls rarer DateTemplate.matchDate now); - standard filters extended with exact prefixed or anchored date templates; template cache introduced (in opposition to default template cache, holds custom templates cached by pattern for possible common usage of same template/regex); --- config/filter.d/apache-badbots.conf | 3 + config/filter.d/apache-fakegooglebot.conf | 2 + config/filter.d/apache-pass.conf | 3 + config/filter.d/assp.conf | 3 +- config/filter.d/dovecot.conf | 5 +- config/filter.d/ejabberd-auth.conf | 4 +- config/filter.d/guacamole.conf | 5 +- config/filter.d/nginx-botsearch.conf | 3 + config/filter.d/nsd.conf | 3 + config/filter.d/php-url-fopen.conf | 3 + config/filter.d/portsentry.conf | 3 + config/filter.d/sogo-auth.conf | 5 +- config/filter.d/squid.conf | 3 + config/filter.d/tine20.conf | 3 + fail2ban/client/jailreader.py | 17 +--- fail2ban/server/datedetector.py | 99 ++++++++++++++----- fail2ban/server/datetemplate.py | 59 +++++++---- fail2ban/server/filter.py | 46 +++------ fail2ban/server/strptime.py | 3 - fail2ban/server/transmitter.py | 2 +- fail2ban/tests/action_d/test_badips.py | 1 + fail2ban/tests/action_d/test_smtp.py | 1 + fail2ban/tests/banmanagertestcase.py | 2 + fail2ban/tests/clientbeautifiertestcase.py | 1 + fail2ban/tests/clientreadertestcase.py | 1 + .../config/filter.d/zzz-generic-example.conf | 5 + fail2ban/tests/datedetectortestcase.py | 59 +++++------ fail2ban/tests/fail2banclienttestcase.py | 1 + fail2ban/tests/fail2banregextestcase.py | 9 ++ fail2ban/tests/failmanagertestcase.py | 1 + fail2ban/tests/files/logs/zzz-generic-example | 8 +- fail2ban/tests/filtertestcase.py | 24 +++++ fail2ban/tests/misctestcase.py | 1 + fail2ban/tests/samplestestcase.py | 1 + fail2ban/tests/servertestcase.py | 8 +- fail2ban/tests/sockettestcase.py | 1 + fail2ban/tests/utils.py | 18 +++- 37 files changed, 280 insertions(+), 136 deletions(-) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index 48b30666..f42aa159 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -14,6 +14,9 @@ failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} + # DEV Notes: # List of bad bots fetched from http://www.user-agents.org # Generated on Thu Nov 7 14:23:35 PST 2013 by files/gen_badbots. diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf index b8a73504..729410ad 100644 --- a/config/filter.d/apache-fakegooglebot.conf +++ b/config/filter.d/apache-fakegooglebot.conf @@ -6,6 +6,8 @@ failregex = ^ .*Googlebot.*$ ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} # DEV Notes: # diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf index 7bdff9c0..3cab87b0 100644 --- a/config/filter.d/apache-pass.conf +++ b/config/filter.d/apache-pass.conf @@ -9,6 +9,9 @@ failregex = ^ - \w+ \[\] "GET HTTP/1\.[01]" 200 \d+ ".*" "[ ignoreregex = +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} + [Init] knocking_url = /knocking/ diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 8d6367c9..7cca81d2 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -20,7 +20,8 @@ failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\ ignoreregex = -datepattern = {^LN-BEG} +datepattern = {^LN-BEG}%%b-%%d-%%Exy %%H:%%M:%%S + {^LN-BEG} # DEV Notes: # V1 Examples matches: diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 136a3947..6f8510fc 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -17,10 +17,11 @@ failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentica ignoreregex = -[Init] - journalmatch = _SYSTEMD_UNIT=dovecot.service +datepattern = {^LN-BEG}TAI64N + {^LN-BEG} + # DEV Notes: # * the first regex is essentially a copy of pam-generic.conf # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016) diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf index 512fdb7c..edd87463 100644 --- a/config/filter.d/ejabberd-auth.conf +++ b/config/filter.d/ejabberd-auth.conf @@ -25,8 +25,6 @@ failregex = ^=INFO REPORT==== ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\ # ignoreregex = -[Init] - # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 @@ -35,3 +33,5 @@ maxlines = 2 # Values: TEXT # journalmatch = + +datepattern = ^(?:=[^=]+={3,} )?({DATE}) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 49cecc5a..09b4e7b0 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -17,6 +17,9 @@ failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" fa # ignoreregex = -[Init] # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 + +datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p + ^WARNING:()** + {^LN-BEG} \ No newline at end of file diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 6853e1e8..0be895b2 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -13,6 +13,9 @@ failregex = ^ \- \S+ \[\] \"(GET|POST|HEAD) \/ \S+\" 404 .+$ ignoreregex = +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + ^[^\[]*\[({DATE}) + {^LN-BEG} # DEV Notes: # Based on apache-botsearch filter diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index 8f32f7be..bfd99544 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -26,3 +26,6 @@ failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ ^%(__prefix_line)sinfo: .* refused, no acl matches\.$ ignoreregex = + +datepattern = {^LN-BEG}Epoch + {^LN-BEG} \ No newline at end of file diff --git a/config/filter.d/php-url-fopen.conf b/config/filter.d/php-url-fopen.conf index 87bd04c8..a7957c9d 100644 --- a/config/filter.d/php-url-fopen.conf +++ b/config/filter.d/php-url-fopen.conf @@ -18,3 +18,6 @@ ignoreregex = # http://blogs.buanzo.com.ar/2009/04/fail2ban-filter-for-php-injection-attacks.html#comment-1489 # # Author: Arturo 'Buanzo' Busleiman + +datepattern = ^[^\[]*\[({DATE}) + {^LN-BEG} diff --git a/config/filter.d/portsentry.conf b/config/filter.d/portsentry.conf index 27dca9b4..35ca2a3d 100644 --- a/config/filter.d/portsentry.conf +++ b/config/filter.d/portsentry.conf @@ -8,5 +8,8 @@ failregex = \/ Port\: [0-9]+ (TCP|UDP) Blocked$ ignoreregex = +datepattern = {^LN-BEG}Epoch + {^LN-BEG} + # Author: Pacop diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf index 1a3d5292..48221dc0 100644 --- a/config/filter.d/sogo-auth.conf +++ b/config/filter.d/sogo-auth.conf @@ -8,7 +8,10 @@ failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '' for user '.*' migh ignoreregex = "^" -datepattern = {^LN-BEG} +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)? + ^[^\[]*\[({DATE}) + {^LN-BEG} # # DEV Notes: diff --git a/config/filter.d/squid.conf b/config/filter.d/squid.conf index e26cab9c..58694c48 100644 --- a/config/filter.d/squid.conf +++ b/config/filter.d/squid.conf @@ -9,5 +9,8 @@ failregex = ^\s+\d\s\s+[A-Z_]+_DENIED/403 .*$ ignoreregex = +datepattern = {^LN-BEG}Epoch + {^LN-BEG} + # Author: Daniel Black diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf index 0fa6eccd..a80d89e8 100644 --- a/config/filter.d/tine20.conf +++ b/config/filter.d/tine20.conf @@ -10,6 +10,9 @@ failregex = ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0 ignoreregex = +datepattern = ^[^-]+ -- [^-]+ -- - ({DATE}) + {^LN-BEG} + # Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766 # Editor: Daniel Black # Advisor: Lars Kneschke diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9d01a693..3849c027 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -112,6 +112,7 @@ class JailReader(ConfigReader): ["string", "ignorecommand", None], ["string", "ignoreip", None], ["string", "filter", ""], + ["string", "datepattern", None], ["string", "action", ""]] # Before interpolation (substitution) add static options always available as default: @@ -195,6 +196,8 @@ class JailReader(ConfigReader): """ stream = [] + if self.__filter: + stream.extend(self.__filter.convert()) for opt, value in self.__opts.iteritems(): if opt == "logpath" and \ not self.__opts.get('backend', None).startswith("systemd"): @@ -216,17 +219,9 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "logencoding", value]) elif opt == "backend": backend = value - elif opt == "maxretry": - stream.append(["set", self.__name, "maxretry", value]) elif opt == "ignoreip": for ip in splitwords(value): stream.append(["set", self.__name, "addignoreip", ip]) - elif opt == "findtime": - stream.append(["set", self.__name, "findtime", value]) - elif opt == "bantime": - stream.append(["set", self.__name, "bantime", value]) - elif opt == "usedns": - stream.append(["set", self.__name, "usedns", value]) elif opt in ("failregex", "ignoreregex"): multi = [] for regex in value.split('\n'): @@ -237,10 +232,8 @@ class JailReader(ConfigReader): stream.append(["multi-set", self.__name, "add" + opt, multi]) elif len(multi): stream.append(["set", self.__name, "add" + opt, multi[0]]) - elif opt == "ignorecommand": - stream.append(["set", self.__name, "ignorecommand", value]) - if self.__filter: - stream.extend(self.__filter.convert()) + elif opt not in ('action', 'filter', 'enabled'): + stream.append(["set", self.__name, opt, value]) for action in self.__actions: if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 7cbc0f5c..cced6d91 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -27,6 +27,7 @@ import time from threading import Lock from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch +from .utils import Utils from ..helpers import getLogger # Gets the instance of the logger. @@ -90,7 +91,7 @@ class DateDetectorCache(object): self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 @@ -112,8 +113,6 @@ class DateDetectorCache(object): # subseconds explicit to avoid possible %m<->%d confusion # with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?") - # TAI64N - self._cacheTemplate(DateTai64n()) # Epoch self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True) self._cacheTemplate(DateEpoch()) @@ -132,6 +131,10 @@ class DateDetectorCache(object): # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + # +00:00 Jan 23 21:59:59.011 2005 + self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + # TAI64N + self._cacheTemplate(DateTai64n()) # self.__templates = self.__tmpcache[0] + self.__tmpcache[1] del self.__tmpcache @@ -168,6 +171,7 @@ class DateDetector(object): templates """ _defCache = DateDetectorCache() + _patternCache = Utils.Cache(maxCount=1000, maxTime=60*60) def __init__(self): self.__templates = list() @@ -183,9 +187,10 @@ class DateDetector(object): # pre-match pattern: self.__preMatch = None - def _appendTemplate(self, template): + def _appendTemplate(self, template, ignoreDup=False): name = template.name if name in self.__known_names: + if ignoreDup: return raise ValueError( "There is already a template with name %s" % name) self.__known_names.add(name) @@ -207,24 +212,56 @@ class DateDetector(object): If a template already exists with the same name. """ if isinstance(template, str): - template = DatePatternRegex(template) + key = pattern = template + if '%' not in pattern: + key = pattern.upper() + template = DateDetector._patternCache.get(key) + + if not template: + if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"): + template = DateEpoch(lineBeginOnly=(key != "EPOCH")) + elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"): + template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False)) + elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): + flt = \ + lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ + lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ + None + self.addDefaultTemplate(flt) + return + elif "{DATE}" in key: + self.addDefaultTemplate( + lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) + return + else: + template = DatePatternRegex(pattern) + + DateDetector._patternCache.set(key, template) + self._appendTemplate(template) + logSys.info(" date pattern `%r`: `%s`", + getattr(template, 'pattern', ''), template.name) + logSys.debug(" date pattern regex for %r: %s", + getattr(template, 'pattern', ''), template.regex) def addDefaultTemplate(self, filterTemplate=None, preMatch=None): """Add Fail2Ban's default set of date templates. """ - for template in sorted(DateDetector._defCache.templates, - lambda a,b: b.hits - a.hits - ): + ignoreDup = len(self.__templates) > 0 + for template in DateDetector._defCache.templates: # filter if specified: if filterTemplate is not None and not filterTemplate(template): continue # if exact pattern available - create copy of template, contains replaced {DATE} with default regex: if preMatch is not None: - regex = getattr(template, 'pattern', template.regex) - template = copy.copy(template) - template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) - # append date detector template: - self._appendTemplate(template) + deftemplate = template + template = DateDetector._patternCache.get((preMatch, deftemplate.name)) + if not template: + regex = getattr(deftemplate, 'pattern', deftemplate.regex) + template = copy.copy(deftemplate) + template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch)) + DateDetector._patternCache.set((preMatch, deftemplate.name), template) + # append date detector template (ignore duplicate if some was added before default): + self._appendTemplate(template, ignoreDup=ignoreDup) @property def templates(self): @@ -250,17 +287,22 @@ class DateDetector(object): The regex match returned from the first successfully matched template. """ - #logSys.log(logLevel, "try to match time for line: %.120s", line) + # if no templates specified - default templates should be used: + if not len(self.__templates): + self.addDefaultTemplate() + logSys.log(logLevel-1, "try to match time for line: %.120s", line) match = None # first try to use last template with same start/end position: + ignoreBySearch = 0x7fffffff i = self.__lastTemplIdx if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template - if template.flags & DateTemplate.LINE_BEGIN: + if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): if logSys.getEffectiveLevel() <= logLevel-1: logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) + ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] if logSys.getEffectiveLevel() <= logLevel-1: @@ -278,18 +320,28 @@ class DateDetector(object): distance = match.start() endpos = match.end() # if different position, possible collision/pattern switch: - if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]: + if ( + template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or + (distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]) + ): logSys.log(logLevel, " matched last time template #%02i", i) else: logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") match = None + else: + logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: self.__lastTemplIdx = 0x7fffffff - logSys.log(logLevel, " search template ...") + logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) found = None, 0x7fffffff, -1 i = 0 for ddtempl in self.__templates: + if logSys.getEffectiveLevel() <= logLevel-1: + logSys.log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) + if i == ignoreBySearch: + i += 1 + continue template = ddtempl.template match = template.matchDate(line) if match: @@ -298,15 +350,18 @@ class DateDetector(object): if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", i, distance, ddtempl.distance, self.__lastPos[0], template.name) + ## last (or single) template - fast stop: + if i+1 >= len(self.__templates): + break ## if line-begin/end anchored - stop searching: if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): break + ## stop searching if next template still unused, but we had already hits: + if (distance == 0 and ddtempl.hits) and not self.__templates[i+1].template.hits: + break ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: - if ( - (distance > ddtempl.distance or distance > self.__lastPos[0]) and - len(self.__templates) > 1 - ): + if distance > ddtempl.distance or distance > self.__lastPos[0]: logSys.log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: @@ -374,7 +429,7 @@ class DateDetector(object): if date is not None: if logSys.getEffectiveLevel() <= logLevel: logSys.log(logLevel, " got time %f for %r using template %s", - date[0], date[1].group(), template.name) + date[0], date[1].group(1), template.name) return date except ValueError: pass diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 70a4b1b5..3a9612aa 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import re +import re, time from abc import abstractmethod from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE @@ -32,9 +32,14 @@ from ..helpers import getLogger logSys = getLogger(__name__) -RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)') -RE_NO_WRD_BOUND_END = re.compile(r'(?(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" + regex = r"((?:^|(?P(?<=^\[))|(?P(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored else: - regex = r"(?P(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))" + regex = r"((?P(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))" self.setRegex(regex, wordBegin='start', wordEnd=True) def getDate(self, line, dateMatch=None): @@ -199,7 +219,7 @@ class DateEpoch(DateTemplate): dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch - return (float(dateMatch.group()), dateMatch) + return (float(dateMatch.group(1)), dateMatch) return None @@ -244,7 +264,13 @@ class DatePatternRegex(DateTemplate): self.setRegex(pattern) def setRegex(self, pattern, wordBegin=True, wordEnd=True): + # original pattern: self._pattern = pattern + # if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin: + if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern): + pattern = RE_EXLINE_BOUND_BEG.sub('', pattern) + wordBegin = 'start' + # wrap to regex: fmt = self._patternRE.sub(r'%(\1)s', pattern) self.name = fmt % self._patternName regex = fmt % timeRE @@ -285,12 +311,11 @@ class DateTai64n(DateTemplate): regex """ - def __init__(self): + def __init__(self, wordBegin=False): DateTemplate.__init__(self) self.name = "TAI64N" # We already know the format for TAI64N - # yoh: we should not add an additional front anchor - self.setRegex("@[0-9a-f]{24}", wordBegin=False) + self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin) def getDate(self, line, dateMatch=None): """Method to return the date for a log line. @@ -310,7 +335,7 @@ class DateTai64n(DateTemplate): dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch - value = dateMatch.group() + value = dateMatch.group(1) seconds_since_epoch = value[2:17] # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cbf59fcb..7f7584cf 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -35,7 +35,6 @@ from .ipdns import DNSUtils, IPAddr from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -94,7 +93,6 @@ class Filter(JailThread): self.ticks = 0 self.dateDetector = DateDetector() - self.dateDetector.addDefaultTemplate() logSys.debug("Created %s" % self) def __repr__(self): @@ -258,34 +256,12 @@ class Filter(JailThread): self.dateDetector = None return else: - key = pattern.upper() - if key == "EPOCH": - template = DateEpoch() - template.name = "Epoch" - elif key == "TAI64N": - template = DateTai64n() - template.name = "TAI64N" - elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"): - self.dateDetector = DateDetector() - flt = \ - lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \ - lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \ - None - self.dateDetector.addDefaultTemplate(flt) - return - elif "{DATE}" in key: - self.dateDetector = DateDetector() - self.dateDetector.addDefaultTemplate( - lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern) - return - else: - template = DatePatternRegex(pattern) - self.dateDetector = DateDetector() - self.dateDetector.appendTemplate(template) - logSys.info(" date pattern `%r`: `%s`", - pattern, template.name) - logSys.debug(" date pattern regex for %r: %s", - pattern, template.regex) + dd = DateDetector() + if not isinstance(pattern, (list, tuple)): + pattern = filter(bool, map(str.strip, re.split('\n+', pattern))) + for pattern in pattern: + dd.appendTemplate(pattern) + self.dateDetector = dd ## # Get the date detector pattern, or Default Detectors if not changed @@ -295,7 +271,8 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: templates = self.dateDetector.templates - if len(templates) > 2: + # lazy template init, by first match + if not len(templates) or len(templates) > 2: return None, "Default Detectors" elif len(templates): if hasattr(templates[0], "pattern"): @@ -303,6 +280,7 @@ class Filter(JailThread): else: pattern = None return pattern, templates[0].name + return None ## # Set the maximum retry value. @@ -483,9 +461,9 @@ class Filter(JailThread): (timeMatch, template) = self.dateDetector.matchTime(l) if timeMatch: tupleLine = ( - l[:timeMatch.start()], - l[timeMatch.start():timeMatch.end()], - l[timeMatch.end():], + l[:timeMatch.start(1)], + l[timeMatch.start(1):timeMatch.end(1)], + l[timeMatch.end(1):], (timeMatch, template) ) else: diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index da00a57e..643984d3 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -55,8 +55,6 @@ timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" # respect possible run in the test-cases (alternate date used there): timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) -# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates: -timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))" def getTimePatternRE(): keys = timeRE.keys() @@ -70,7 +68,6 @@ def getTimePatternRE(): 'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek", 'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%", 'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name", - 'ExLB': '{^LN-BEG}', } for key in set(keys) - set(names): # may not have them all... if key.startswith('Ex'): diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2f5be043..a0dfc639 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -303,7 +303,7 @@ class Transmitter: actionvalue = command[4] setattr(action, actionkey, actionvalue) return getattr(action, actionkey) - raise Exception("Invalid command (no set action or not yet implemented)") + raise Exception("Invalid command %r (no set action or not yet implemented)" % (command[1],)) def __commandGet(self, command): name = command[0] diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index 64db44e7..2f3b6723 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -32,6 +32,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable def setUp(self): """Call before every test case.""" + super(BadIPsActionTest, self).setUp() unittest.F2B.SkipIfNoNetwork() self.jail = DummyJail() diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 5c8b1923..dc28d5c0 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -45,6 +45,7 @@ class SMTPActionTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(SMTPActionTest, self).setUp() self.jail = DummyJail() pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py") pythonModuleName = os.path.basename(pythonModule.rstrip(".py")) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 4d964425..2c9d6601 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -32,6 +32,7 @@ from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(AddFailure, self).setUp() self.__ticket = BanTicket('193.168.0.128', 1167605999.0) self.__banManager = BanManager() @@ -134,6 +135,7 @@ class AddFailure(unittest.TestCase): class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() self.__ban_ip = "93.184.216.34" self.__asn = "15133" diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py index 0390a8ff..7d20e84e 100644 --- a/fail2ban/tests/clientbeautifiertestcase.py +++ b/fail2ban/tests/clientbeautifiertestcase.py @@ -32,6 +32,7 @@ class BeautifierTest(unittest.TestCase): def setUp(self): """ Call before every test case """ + super(BeautifierTest, self).setUp() self.b = Beautifier() def tearDown(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e68523c2..36722ff1 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -55,6 +55,7 @@ class ConfigReaderTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(ConfigReaderTest, self).setUp() self.d = tempfile.mkdtemp(prefix="f2b-temp") self.c = ConfigReaderUnshared(basedir=self.d) diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf index df30d725..8d1f2ae9 100644 --- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf +++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf @@ -20,3 +20,8 @@ failregex = ^%(__prefix_line)sF2B: failure from $ # just to test multiple ignoreregex: ignoreregex = ^%(__prefix_line)sF2B: error from 192.0.2.251$ ^%(__prefix_line)sF2B: error from 192.0.2.252$ + +# specify only exact date patterns, +1 with %%Y to test usage of last known date by wrong dates like 0000-00-00... +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)? + {^LN-BEG}%%Y(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 6bf92582..fdb7ec71 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -58,6 +58,8 @@ class DateDetectorTest(LogCaptureTestCase): return self.__datedetector def testGetEpochTime(self): + self.__datedetector = DateDetector() + self.__datedetector.appendTemplate('EPOCH') # correct epoch time, using all variants: for dateUnix in (1138049999, 32535244799): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): @@ -67,7 +69,7 @@ class DateDetectorTest(LogCaptureTestCase): self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) - self.assertIn(matchlog.group(), (str(dateUnix), str(dateUnix)+'.555')) + self.assertIn(matchlog.group(1), (str(dateUnix), str(dateUnix)+'.555')) # wrong, no epoch time (< 10 digits, more as 11 digits, begin/end of word) : for dateUnix in ('123456789', '9999999999999999', '1138049999A', 'A1138049999'): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): @@ -85,7 +87,7 @@ class DateDetectorTest(LogCaptureTestCase): # of fail2ban -- we just ignore incorrect day of the week ( datelog, matchlog ) = self.datedetector.getTime(log) self.assertEqual(datelog, dateUnix) - self.assertEqual(matchlog.group(), 'Jan 23 21:59:59') + self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand @@ -150,7 +152,7 @@ class DateDetectorTest(LogCaptureTestCase): ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) - self.assertEqual(logMatch.group(), rdate) + self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) @@ -164,7 +166,7 @@ class DateDetectorTest(LogCaptureTestCase): ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) - self.assertEqual(logMatch.group(), rdate) + self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) @@ -183,23 +185,23 @@ class DateDetectorTest(LogCaptureTestCase): self.assertNotEqual(logdate, None) ( logTime, logMatch ) = logdate self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') + self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end for i in xrange(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '11/10/2012 02:37:17') + self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') # and now back to the original ( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) - self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') + self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') def testDateTemplate(self): t = DateTemplate() t.setRegex('^a{3,5}b?c*$') - self.assertEqual(t.regex, '^a{3,5}b?c*$') + self.assertEqual(t.regex, '^(a{3,5}b?c*)$') self.assertRaises(Exception, t.getDate, '') - self.assertEqual(t.matchDate('aaaac').group(), 'aaaac') + self.assertEqual(t.matchDate('aaaac').group(1), 'aaaac') ## no word boundaries left and right: t = DatePatternRegex() @@ -208,22 +210,22 @@ class DateDetectorTest(LogCaptureTestCase): self.assertFalse('**' in t.regex) # match date: dt = 'TIME:20050102T010203' - self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt) - self.assertEqual(t.matchDate(dt).group(), dt) + self.assertEqual(t.matchDate('X' + dt + 'X').group(1), dt) + self.assertEqual(t.matchDate(dt).group(1), dt) # wrong year (for exact %ExY): dt = 'TIME:50050102T010203' self.assertFalse(t.matchDate(dt)) - ## start boundary left and word boundary right: + ## start boundary left and word boundary right (automatically if not **): t = DatePatternRegex() - t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS' + t.pattern = '{^LN-BEG}time:%ExY%Exm%ExdT%ExH%ExM%ExS' self.assertTrue('^' in t.regex) # try match date: dt = 'time:20050102T010203' self.assertFalse(t.matchDate('X' + dt)) self.assertFalse(t.matchDate(dt + 'X')) - self.assertEqual(t.matchDate('##' + dt + '...').group(), dt) - self.assertEqual(t.matchDate(dt).group(), dt) + self.assertEqual(t.matchDate('##' + dt + '...').group(1), dt) + self.assertEqual(t.matchDate(dt).group(1), dt) # case sensitive: dt = 'TIME:20050102T010203' self.assertFalse(t.matchDate(dt)) @@ -232,9 +234,9 @@ class DateDetectorTest(LogCaptureTestCase): t = DatePatternRegex() t.pattern = '^%Y %b %d' self.assertTrue('(?iu)' in t.regex) - dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt) - dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt) - dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt) + dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt) def testAmbiguousInOrderedTemplates(self): dd = self.datedetector @@ -259,7 +261,7 @@ class DateDetectorTest(LogCaptureTestCase): logSys.debug('Line: %s', line) match, template = dd.matchTime(line) self.assertTrue(match) - self.assertEqual(match.group(), debit) + self.assertEqual(match.group(1), debit) def testLowLevelLogging(self): # test coverage for the deep (heavy) debug messages: @@ -363,31 +365,30 @@ class CustomDateFormatsTest(unittest.TestCase): ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), - # Explicit bound in start of the line using %ExLB key, + # Explicit bound in start of the line using {^LN-BEG} key, # (negative) in the 1st case without line begin boundary - wrong date may be found, # (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found) # (positive) and in 3th case with line begin boundary - find the correct date ("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), - (None, "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), - ("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), - # Explicit bound in start of the line using %ExLB key, + (None, "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), + ("20031230 010203", "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), + # Explicit bound in start of the line using {^LN-BEG} key, # up to 2 non-alphanumeric chars front, ** - no word boundary on the right - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), - ("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), + ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: dd = defDD else: - dp = DatePatternRegex(dp) dd = DateDetector() dd.appendTemplate(dp) date = dd.getTime(line) if matched: self.assertTrue(date) - self.assertEqual(matched, date[1].group()) + self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(date, None) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index e212ff23..8e25fd77 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -743,6 +743,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "maxretry = 3", "findtime = 10m", "failregex = ^\s*failure (401|403) from ", + "datepattern = {^LN-BEG}EPOCH", "", "[test-jail1]", "backend = " + backend, "filter =", "action = ", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index c62377a9..5767a3f7 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -101,6 +101,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWrongIngnoreRE(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "{^LN-BEG}EPOCH", "test", r".*? from $", r".**" ) self.assertFalse(fail2banRegex.start(opts, args)) @@ -108,6 +109,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectFound(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--print-no-missed", "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", r"Authentication failure for .*? from $" @@ -136,6 +138,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -151,6 +154,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -160,6 +164,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw_noDns(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", "--usedns=no", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -169,6 +174,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_2(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 @@ -178,6 +184,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testVerbose(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--verbose", "--verbose-date", "--print-no-missed", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 @@ -190,6 +197,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) self.assertTrue(fail2banRegex.start(opts, args)) @@ -203,6 +211,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronCharDebuggex(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 6e7bf367..ed68e2a8 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -36,6 +36,7 @@ class AddFailure(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(AddFailure, self).setUp() self.__items = None self.__failManager = FailManager() diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index 51d3974c..2044c387 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (using precise year pattern): -# failJSON: { "match": false} +# -- wrong time direct in journal-line (used last known date): +# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (using precise year pattern): -# failJSON: { "match": false} +# -- wrong time direct in journal-line (used last known date): +# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" } diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ea996072..227050f2 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -270,6 +270,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p class BasicFilter(unittest.TestCase): def setUp(self): + super(BasicFilter, self).setUp() self.filter = Filter('name') def testGetSetUseDNS(self): @@ -363,6 +364,7 @@ class IgnoreIP(LogCaptureTestCase): setUpMyTime() self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addFailRegex('') + self.filter.setDatePattern('{^LN-BEG}EPOCH') self.filter.processLineAndAdd('1387203300.222 192.168.1.32') self.assertLogged('Ignore 192.168.1.32') tearDownMyTime() @@ -461,6 +463,7 @@ class LogFileFilterPoll(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(LogFileFilterPoll, self).setUp() self.filter = FilterPoll(DummyJail()) self.filter.addLogPath(LogFileFilterPoll.FILENAME) @@ -653,6 +656,8 @@ class LogFileMonitor(LogCaptureTestCase): self.assertLogged('Unable to open %s' % self.name) def testErrorProcessLine(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.filter.sleeptime /= 1000.0 ## produce error with not callable processLine: _org_processLine = self.filter.processLine @@ -715,6 +720,8 @@ class LogFileMonitor(LogCaptureTestCase): pass def testNewChangeViaGetFailures_simple(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # suck in lines from this sample log file self.filter.getFailures(self.name) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) @@ -730,6 +737,8 @@ class LogFileMonitor(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_rewrite(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we rewrite the file at once self.file.close() @@ -748,6 +757,8 @@ class LogFileMonitor(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_move(self): + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we move file into a new location while it has been open already self.file.close() @@ -769,6 +780,7 @@ class CommonMonitorTestCase(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(CommonMonitorTestCase, self).setUp() self._failTotal = 0 def waitFailTotal(self, count, delay=1.): @@ -819,6 +831,8 @@ def get_monitor_failures_testcase(Filter_): self.jail = DummyJail() self.filter = Filter_(self.jail) self.filter.addLogPath(self.name, autoSeek=False) + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") self.filter.start() @@ -1223,6 +1237,8 @@ class GetFailures(LogCaptureTestCase): self.jail = DummyJail() self.filter = FileFilter(self.jail) self.filter.active = True + # speedup search using exact date pattern: + self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # TODO Test this #self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") #self.filter.setTimePattern("%b %d %H:%M:%S") @@ -1329,6 +1345,11 @@ class GetFailures(LogCaptureTestCase): output = (('212.41.96.186', 4, 1124013600.0), ('212.41.96.185', 2, 1124013598.0)) + # speedup search using exact date pattern: + self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', + '^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', + '^EPOCH' + )) self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0) self.filter.addFailRegex("Invalid user .* ") @@ -1358,6 +1379,8 @@ class GetFailures(LogCaptureTestCase): if enc is not None: self.tearDown();self.setUp(); self.filter.setLogEncoding(enc); + # speedup search using exact date pattern: + self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) @@ -1533,6 +1556,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(DNSUtilsNetworkTests, self).setUp() unittest.F2B.SkipIfNoNetwork() def test_IPAddr(self): diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 908e4f6c..cd841a0f 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -86,6 +86,7 @@ def _getSysPythonVersion(): class SetupTest(unittest.TestCase): def setUp(self): + super(SetupTest, self).setUp() unittest.F2B.SkipIfFast() setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 31b1812e..0ed3e764 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -43,6 +43,7 @@ class FilterSamplesRegex(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(FilterSamplesRegex, self).setUp() self.filter = Filter(None) self.filter.active = True diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index cd7ce5c9..03fd1a58 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -65,7 +65,7 @@ class TransmitterBase(unittest.TestCase): def setUp(self): """Call before every test case.""" - #super(TransmitterBase, self).setUp() + super(TransmitterBase, self).setUp() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -301,9 +301,11 @@ class Transmitter(TransmitterBase): ("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"), jail=self.jailName) self.setGetTest( - "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) + "datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName) self.setGetTest( - "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) + "datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName) + self.setGetTest( + "datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self): diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 5bf0be57..1a94a952 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -41,6 +41,7 @@ class Socket(unittest.TestCase): def setUp(self): """Call before every test case.""" + super(Socket, self).setUp() self.server = AsyncServer(self) sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'socket') os.close(sock_fd) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 22079456..fc60f87b 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -526,6 +526,16 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'): self.fail(msg) unittest.TestCase.assertNotIn = assertNotIn +_org_setUp = unittest.TestCase.setUp +def _customSetUp(self): + # print('=='*10, self) + if unittest.F2B.log_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! + print("") + logSys.debug('='*10 + ' %s ' + '='*20, self.id()) + _org_setUp(self) + +unittest.TestCase.setUp = _customSetUp + class LogCaptureTestCase(unittest.TestCase): @@ -601,12 +611,11 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy) logSys.handlers = [self._log] - if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! - print("") + if self._old_level <= logging.DEBUG: logSys.handlers += self._old_handlers - logSys.debug('='*10 + ' %s ' + '='*20, self.id()) - else: + else: # lowest log level to capture messages logSys.setLevel(logging.DEBUG) + super(LogCaptureTestCase, self).setUp() def tearDown(self): """Call after every test case.""" @@ -615,6 +624,7 @@ class LogCaptureTestCase(unittest.TestCase): logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level + super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): logged = self._log.getvalue()