From cc311b56f3add83e38d444969c2ddab618f6bf36 Mon Sep 17 00:00:00 2001 From: benrubson Date: Fri, 23 Dec 2016 22:57:24 +0100 Subject: [PATCH 01/76] Apache URIs can contain spaces --- config/filter.d/apache-auth.conf | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 8a63858d..36e5aed7 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -10,18 +10,18 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$ - ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ - ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*?: password mismatch: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01631: )?user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$ +failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration + ^%(_apache_error_client)s (AH01617: )?user \S* authentication failure + ^%(_apache_error_client)s (AH01618: )?user \S* not found + ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme + ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S* to access .* failed + ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user \S*: password mismatch + ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `\S*' in realm `.+' (not found|denied by provider) + ^%(_apache_error_client)s (AH01631: )?user \S*: authorization failure ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+(, referer: \S+)?\s*$ ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01793: )?invalid qop `.*?' received: \S*(, referer: \S+)?\s*$ + ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm ` + ^%(_apache_error_client)s (AH01793: )?invalid qop ` ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$ ignoreregex = @@ -43,6 +43,8 @@ ignoreregex = # all of these expressions. Lots of submodules like mod_authz_* return back to mod_authz_core # to return the actual failure. # +# Note that URI can contain spaces. +# # See also: http://wiki.apache.org/httpd/ListOfErrors # Expressions that don't have tests and aren't common. # more be added with https://issues.apache.org/bugzilla/show_bug.cgi?id=55284 From 97d417926dc1e9eef637c3ba3b45d3c48814b0e0 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Mar 2017 17:56:46 +0100 Subject: [PATCH 02/76] repairs testing of missing samples for all regex after filter settings (mode) changed --- fail2ban/server/filter.py | 9 +++---- fail2ban/tests/samplestestcase.py | 44 ++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6b782fcc..ca2dae86 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -183,15 +183,12 @@ class Filter(JailThread): "valid", index) ## - # Get the regular expression which matches the failure. + # Get the regular expressions as list. # - # @return the regular expression + # @return the regular expression list def getFailRegex(self): - failRegex = list() - for regex in self.__failRegex: - failRegex.append(regex.getRegex()) - return failRegex + return [regex.getRegex() for regex in self.__failRegex] ## # Add the regular expression which matches the failure. diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 1d33cc44..00a8f305 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -99,8 +99,8 @@ class FilterSamplesRegex(unittest.TestCase): optval = opt[3] elif opt[0] == 'set': optval = [opt[3]] - else: - continue + else: # pragma: no cover - unexpected + self.fail('Unexpected config-token %r in stream' % (opt,)) for optval in optval: if opt[2] == "prefregex": self.filter.prefRegex = optval @@ -115,11 +115,13 @@ class FilterSamplesRegex(unittest.TestCase): # test regexp contains greedy catch-all before , that is # not hard-anchored at end or has not precise sub expression after : - for fr in self.filter.getFailRegex(): + regexList = self.filter.getFailRegex() + for fr in regexList: if RE_WRONG_GREED.search(fr): # pragma: no cover raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before , " "that is not hard-anchored at end or has not precise sub expression after :\n%s" % (name, str(fr).replace(RE_HOST, ''))) + return regexList def testSampleRegexsFactory(name, basedir): def testFilter(self): @@ -128,8 +130,17 @@ def testSampleRegexsFactory(name, basedir): os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), "No sample log file available for '%s' filter" % name) - regexsUsed = set() + regexList = None + regexsUsedIdx = set() + regexsUsedRe = set() filenames = [name] + + def _testMissingSamples(): + for failRegexIndex, failRegex in enumerate(regexList): + self.assertTrue( + failRegexIndex in regexsUsedIdx or failRegex in regexsUsedRe, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) i = 0 while i < len(filenames): filename = filenames[i]; i += 1; @@ -143,25 +154,30 @@ def testSampleRegexsFactory(name, basedir): faildata = json.loads(jsonREMatch.group(2)) # filterOptions - dict in JSON to control filter options (e. g. mode, etc.): if jsonREMatch.group(1) == 'filterOptions': + # another filter mode - we should check previous also: + if self.filter is not None: + _testMissingSamples() + regexsUsedIdx = set() # clear used indices (possible overlapping by mode change) + # read filter with another setting: self.filter = None - self._readFilter(name, basedir, opts=faildata) + regexList = self._readFilter(name, basedir, opts=faildata) continue # addFILE - filename to "include" test-files should be additionally parsed: if jsonREMatch.group(1) == 'addFILE': filenames.append(faildata) continue # failJSON - faildata contains info of the failure to check it. - except ValueError as e: + except ValueError as e: # pragma: no cover - we've valid json's raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) line = next(logFile) elif line.startswith("#") or not line.strip(): continue - else: + else: # pragma: no cover - normally unreachable faildata = {} if self.filter is None: - self._readFilter(name, basedir, opts=None) + regexList = self._readFilter(name, basedir, opts=None) try: ret = self.filter.processLine(line) @@ -174,7 +190,8 @@ def testSampleRegexsFactory(name, basedir): failregex, fid, fail2banTime, fail = ret[0] # Bypass no failure helpers-regexp: if not faildata.get('match', False) and (fid is None or fail.get('nofail')): - regexsUsed.add(failregex) + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) continue # Check line is flagged to match @@ -208,16 +225,13 @@ def testSampleRegexsFactory(name, basedir): jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), fail2banTime - jsonTime) ) - regexsUsed.add(failregex) + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover raise AssertionError("%s on: %s:%i, line:\n%s" % ( e, logFile.filename(), logFile.filelineno(), line)) - for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): - self.assertTrue( - failRegexIndex in regexsUsed, - "Regex for filter '%s' has no samples: %i: %r" % - (name, failRegexIndex, failRegex)) + _testMissingSamples() return testFilter From 5561423be3b2d4636f5484183c3ad470fd326d06 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Mar 2017 18:00:53 +0100 Subject: [PATCH 03/76] filter.d/sshd.conf: fixed failregex format - some parts are optional, new ddos more precise rule (Connection reset by with host entry); closes gh-1719 --- config/filter.d/sshd.conf | 7 ++++--- .../tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 5 +++-- fail2ban/tests/files/logs/sshd | 7 ++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 8163aa03..320ab59c 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -41,17 +41,18 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ - ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ + ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s - ^Disconnecting: Too many authentication failures for .+?%(__suff)s + ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s ^Received disconnect from : 11: ^Connection closed by %(__suff)s$ mdre-normal = mdre-ddos = ^Did not receive identification string from %(__suff)s$ + ^Connection reset by %(__on_port_opt)s%(__suff)s ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: - ^Read from socket failed: Connection reset by peer \[preauth\] + ^Read from socket failed: Connection reset by peer%(__suff)s mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 4f28e60f..d6eecd4b 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -40,12 +40,13 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from : 11: .+%(__suff)s$ - ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ + ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s%(__prefix_line_ml2)sConnection closed by %(__suff)s$ + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ + ^%(__prefix_line_sl)sConnection reset by %(__on_port_opt)s%(__suff)s ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index b53b3d96..fe19591c 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -168,7 +168,7 @@ Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication fa # failJSON: { "match": false } Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 on 127.0.0.1 port 22 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" } -Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] +Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures [preauth] # failJSON: { "time": "2004-11-23T21:50:37", "match": true , "host": "61.0.0.1", "desc": "New logline format as openssh 6.8 to replace prev multiline version" } Nov 23 21:50:37 myhost sshd[21810]: error: maximum authentication attempts exceeded for root from 61.0.0.1 port 49940 ssh2 [preauth] @@ -208,6 +208,11 @@ Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1- # failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" } Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] +# gh-1719: +# failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } +Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] + + # filterOptions: {"mode": "extra"} # several other cases from gh-864: From 93ec9e01d402d9fcbb83f4548de1eab0bf581f13 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 17 Mar 2017 10:03:34 +0100 Subject: [PATCH 04/76] fixes a small blemish by output in beautifier; command "unban" returns a count of tickets that were flushed --- fail2ban/client/beautifier.py | 7 ++++++- fail2ban/server/transmitter.py | 8 ++++---- fail2ban/tests/fail2banclienttestcase.py | 8 ++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index df44afbb..4d9e549f 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -89,6 +89,8 @@ class Beautifier: val = " ".join(map(str, res1[1])) if isinstance(res1[1], list) else res1[1] msg.append("%s %s:\t%s" % (prefix1, res1[0], val)) msg = "\n".join(msg) + elif len(inC) < 2: + pass # to few cmd args for below elif inC[1] == "syslogsocket": msg = "Current syslog socket is:\n" msg += "`- " + response @@ -110,6 +112,8 @@ class Beautifier: else: msg = "Current database purge age is:\n" msg += "`- %iseconds" % response + elif len(inC) < 3: + pass # to few cmd args for below elif inC[2] in ("logpath", "addlogpath", "dellogpath"): if len(response) == 0: msg = "No file is currently monitored" @@ -178,7 +182,8 @@ class Beautifier: msg += ", ".join(response) except Exception: logSys.warning("Beautifier error. Please report the error") - logSys.error("Beautify %r with %r failed", response, self.__inputCmd) + logSys.error("Beautify %r with %r failed", response, self.__inputCmd, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) msg = repr(msg) + repr(response) return msg diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 265b9704..ad21b851 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -108,11 +108,11 @@ class Transmitter: value = command[1:] # if all ips: if len(value) == 1 and value[0] == "--all": - self.__server.setUnbanIP() - return + return self.__server.setUnbanIP() + cnt = 0 for value in value: - self.__server.setUnbanIP(None, value) - return None + cnt += self.__server.setUnbanIP(None, value) + return cnt elif command[0] == "echo": return command[1:] elif command[0] == "sleep": diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index adaf4719..683368ee 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1075,6 +1075,14 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail1] Ban 192.0.2.4", all=True ) + # unban all (just to test command, already empty - nothing to unban): + self.pruneLog("[test-phase 7b]") + self.execSuccess(startparams, + "--async", "unban", "--all") + self.assertLogged( + "Flush ban list", + "Unbanned 0, 0 ticket(s) in 'test-jail1'", all=True) + # backend-switch (restart instead of reload): self.pruneLog("[test-phase 8a]") _write_jail_cfg(enabled=[1], backend="xxx-unknown-backend-zzz") From e5c9f9ec1cfa27180ce1f0f62036fe4fd42316b3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Mar 2017 16:35:40 +0100 Subject: [PATCH 05/76] [interim commit] try to fix possible escape vulnerability in actions --- fail2ban/server/action.py | 67 +++++++++++++++++++++----------- fail2ban/server/actions.py | 1 + fail2ban/server/ipdns.py | 5 +++ fail2ban/server/utils.py | 64 +++++++++++++++++++++--------- fail2ban/tests/actiontestcase.py | 28 ++++++++++--- 5 files changed, 119 insertions(+), 46 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 2a773638..bdb25d27 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -453,7 +453,7 @@ class CommandAction(ActionBase): return value @classmethod - def replaceTag(cls, query, aInfo, conditional='', cache=None, substRec=True): + def replaceTag(cls, query, aInfo, conditional='', cache=None): """Replaces tags in `query` with property values. Parameters @@ -481,9 +481,8 @@ class CommandAction(ActionBase): # **Important**: don't replace if calling map - contains dynamic values only, # no recursive tags, otherwise may be vulnerable on foreign user-input: noRecRepl = isinstance(aInfo, CallingMap) - if noRecRepl: - subInfo = aInfo - else: + subInfo = aInfo + if not noRecRepl: # substitute tags recursive (and cache if possible), # first try get cached tags dictionary: subInfo = csubkey = None @@ -534,13 +533,50 @@ class CommandAction(ActionBase): "unexpected too long replacement interpolation, " "possible self referencing definitions in query: %s" % (query,)) - # cache if possible: if cache is not None: cache[ckey] = value # return value + @classmethod + def replaceDynamicTags(cls, realCmd, aInfo): + """Replaces dynamical tags in `query` with property values. + + **Important** + ------------- + Because this tags are dynamic resp. foreign (user) input: + - values should be escaped + - no recursive substitution (no interpolation for >) + - don't use cache + + Parameters + ---------- + query : str + String with tags. + aInfo : dict + Tags(keys) and associated values for substitution in query. + + Returns + ------- + str + shell script as string or array with tags replaced (direct or as variables). + """ + realCmd = cls.replaceTag(realCmd, aInfo, conditional=False) + # Replace ticket options (filter capture groups) non-recursive: + if '<' in realCmd: + tickData = aInfo.get("F-*") + if not tickData: tickData = {} + def substTag(m): + tn = mapTag2Opt(m.groups()[0]) + try: + return str(tickData[tn]) + except KeyError: + return "" + + realCmd = FCUSTAG_CRE.sub(substTag, realCmd) + return realCmd + def _processCmd(self, cmd, aInfo=None, conditional=''): """Executes a command with preliminary checks and substitutions. @@ -605,21 +641,9 @@ class CommandAction(ActionBase): realCmd = self.replaceTag(cmd, self._properties, conditional=conditional, cache=self.__substCache) - # Replace dynamical tags (don't use cache here) + # Replace dynamical tags, important - don't cache, no recursion and auto-escape here if aInfo is not None: - realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional) - # Replace ticket options (filter capture groups) non-recursive: - if '<' in realCmd: - tickData = aInfo.get("F-*") - if not tickData: tickData = {} - def substTag(m): - tn = mapTag2Opt(m.groups()[0]) - try: - return str(tickData[tn]) - except KeyError: - return "" - - realCmd = FCUSTAG_CRE.sub(substTag, realCmd) + realCmd = self.replaceDynamicTags(realCmd, aInfo) else: realCmd = cmd @@ -653,8 +677,5 @@ class CommandAction(ActionBase): logSys.debug("Nothing to do") return True - _cmd_lock.acquire() - try: + with _cmd_lock: return Utils.executeCmd(realCmd, timeout, shell=True, output=False, **kwargs) - finally: - _cmd_lock.release() diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index e0719cde..3a85e569 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -290,6 +290,7 @@ class Actions(JailThread, Mapping): AI_DICT = { "ip": lambda self: self.__ticket.getIP(), + "family": lambda self: self['ip'].familyStr, "ip-rev": lambda self: self['ip'].getPTR(''), "ip-host": lambda self: self['ip'].getHost(), "fid": lambda self: self.__ticket.getID(), diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 8990618a..bd3b812d 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -261,6 +261,11 @@ class IPAddr(object): def family(self): return self._family + FAM2STR = {socket.AF_INET: 'inet4', socket.AF_INET6: 'inet6'} + @property + def familyStr(self): + return IPAddr.FAM2STR.get(self._family) + @property def plen(self): return self._plen diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index b258ae77..56428294 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -28,7 +28,7 @@ import signal import subprocess import sys import time -from ..helpers import getLogger, uni_decode +from ..helpers import getLogger, _merge_dicts, uni_decode if sys.version_info >= (3, 3): import importlib.machinery @@ -116,7 +116,22 @@ class Utils(): return flags @staticmethod - def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, success_codes=(0,)): + def buildShellCmd(realCmd, varsDict): + # build map as array of vars and command line array: + varsStat = "" + if not isinstance(realCmd, list): + realCmd = [realCmd] + i = len(realCmd)-1 + for k, v in varsDict.iteritems(): + varsStat += "%s=$%s " % (k, i) + realCmd.append(v) + i += 1 + realCmd[0] = varsStat + "\n" + realCmd[0] + return realCmd + + @staticmethod + def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, + success_codes=(0,), varsDict=None): """Executes a command. Parameters @@ -131,6 +146,8 @@ class Utils(): output : bool If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode). If False, just indication of success is returned + varsDict: dict + variables supplied to the command (or to the shell script) Returns ------- @@ -146,10 +163,18 @@ class Utils(): """ stdout = stderr = None retcode = None - popen = None + popen = env = None + if varsDict: + if shell: + # build map as array of vars and command line array: + realCmd = Utils.buildShellCmd(realCmd, varsDict) + else: # pragma: no cover - currently unused + env = _merge_dicts(os.environ, varsDict) + realCmdId = id(realCmd) + outCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd) try: popen = subprocess.Popen( - realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, + realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env, preexec_fn=os.setsid # so that killpg does not kill our process ) # wait with timeout for process has terminated: @@ -158,13 +183,15 @@ class Utils(): def _popen_wait_end(): retcode = popen.poll() return (True, retcode) if retcode is not None else None - retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL) + # popen.poll is fast operation so we can put down the sleep interval: + retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL / 100) if retcode: retcode = retcode[1] # if timeout: if retcode is None: - logSys.error("%s -- timed out after %s seconds." % - (realCmd, timeout)) + if outCmd: outCmd(logging.ERROR); outCmd = None + logSys.error("%x -- timed out after %s seconds." % + (realCmdId, timeout)) pgid = os.getpgid(popen.pid) # if not tree - first try to terminate and then kill, otherwise - kill (-9) only: os.killpg(pgid, signal.SIGTERM) # Terminate the process @@ -185,48 +212,49 @@ class Utils(): return False if not output else (False, stdout, stderr, retcode) std_level = logging.DEBUG if retcode in success_codes else logging.ERROR + if std_level > logSys.getEffectiveLevel(): + if outCmd: outCmd(std_level-1); outCmd = None # if we need output (to return or to log it): if output or std_level >= logSys.getEffectiveLevel(): + # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode. if popen.stdout: try: if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stdout, False) stdout = popen.stdout.read() - except IOError as e: + except IOError as e: # pragma: no cover logSys.error(" ... -- failed to read stdout %s", e) if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): - logSys.log(std_level, "%s -- stdout:", realCmd) for l in stdout.splitlines(): - logSys.log(std_level, " -- stdout: %r", uni_decode(l)) + logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l)) popen.stdout.close() if popen.stderr: try: if retcode is None or retcode < 0: Utils.setFBlockMode(popen.stderr, False) stderr = popen.stderr.read() - except IOError as e: + except IOError as e: # pragma: no cover logSys.error(" ... -- failed to read stderr %s", e) if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): - logSys.log(std_level, "%s -- stderr:", realCmd) for l in stderr.splitlines(): - logSys.log(std_level, " -- stderr: %r", uni_decode(l)) + logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) popen.stderr.close() success = False if retcode in success_codes: - logSys.debug("%-.40s -- returned successfully %i", realCmd, retcode) + logSys.debug("%x -- returned successfully %i", realCmdId, retcode) success = True elif retcode is None: - logSys.error("%-.40s -- unable to kill PID %i", realCmd, popen.pid) + logSys.error("%x -- unable to kill PID %i", realCmdId, popen.pid) elif retcode < 0 or retcode > 128: # dash would return negative while bash 128 + n sigcode = -retcode if retcode < 0 else retcode - 128 - logSys.error("%-.40s -- killed with %s (return code: %s)", - realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode) + logSys.error("%x -- killed with %s (return code: %s)", + realCmdId, signame.get(sigcode, "signal %i" % sigcode), retcode) else: msg = _RETCODE_HINTS.get(retcode, None) - logSys.error("%-.40s -- returned %i", realCmd, retcode) + logSys.error("%x -- returned %i", realCmdId, retcode) if msg: logSys.info("HINT on %i: %s", retcode, msg % locals()) if output: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 70562baf..dc847290 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -389,6 +389,23 @@ class CommandActionTest(LogCaptureTestCase): self.assertLogged('Nothing to do') self.pruneLog() + def testExecuteWithVars(self): + self.assertTrue(self.__action.executeCmd( + r'''printf %b "foreign input:\n''' + r''' -- $f2bV_A --\n''' + r''' -- $f2bV_B --\n''' + r''' -- $(echo $f2bV_C) --''' # echo just replaces \n to test it as single line + r'''"''', + varsDict={ + 'f2bV_A': 'I\'m a hacker; && $(echo $f2bV_B)', + 'f2bV_B': 'I"m very bad hacker', + 'f2bV_C': '`Very | very\n$(bad & worst hacker)`' + })) + self.assertLogged(r"""foreign input:""", + ' -- I\'m a hacker; && $(echo $f2bV_B) --', + ' -- I"m very bad hacker --', + ' -- `Very | very $(bad & worst hacker)` --', all=True) + def testExecuteIncorrectCmd(self): CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null') self.assertLogged('HINT on 127: "Command not found"') @@ -400,8 +417,9 @@ class CommandActionTest(LogCaptureTestCase): self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=timeout)) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + timeout and time.time() <= stime + timeout + 1) - self.assertLogged('sleep 30 -- timed out after') - self.assertLogged('sleep 30 -- killed with SIGTERM') + self.assertLogged('sleep 30', ' -- timed out after', all=True) + self.assertLogged(' -- killed with SIGTERM', + ' -- killed with SIGKILL') def testExecuteTimeoutWithNastyChildren(self): # temporary file for a nasty kid shell script @@ -457,9 +475,9 @@ class CommandActionTest(LogCaptureTestCase): # Verify that the process itself got killed self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3)) self.assertLogged('my pid ', 'Resource temporarily unavailable') - self.assertLogged('timed out') - self.assertLogged('killed with SIGTERM', - 'killed with SIGKILL') + self.assertLogged(' -- timed out') + self.assertLogged(' -- killed with SIGTERM', + ' -- killed with SIGKILL') os.unlink(tmpFilename) os.unlink(tmpFilename + '.pid') From 4f1473724bdb074f7f271a0dcc18ff8a928aaf2f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Mar 2017 20:58:01 +0100 Subject: [PATCH 06/76] fixed grave vulnerability by wrong escape of tags by executing of shell actions --- fail2ban/server/action.py | 44 +++++++++++++++++++++++++++++--- fail2ban/server/utils.py | 5 ++-- fail2ban/tests/actiontestcase.py | 30 +++++++++++++++++++++- fail2ban/tests/servertestcase.py | 6 ++--- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index bdb25d27..1c5eb0c9 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -539,6 +539,9 @@ class CommandAction(ActionBase): # return value + ESCAPE_CRE = re.compile(r"""[\\#&;`|*?~<>\^\(\)\[\]{}$'"\n\r]""") + ESCAPE_VN_CRE = re.compile(r"\W") + @classmethod def replaceDynamicTags(cls, realCmd, aInfo): """Replaces dynamical tags in `query` with property values. @@ -546,7 +549,7 @@ class CommandAction(ActionBase): **Important** ------------- Because this tags are dynamic resp. foreign (user) input: - - values should be escaped + - values should be escaped (using "escape" as shell variable) - no recursive substitution (no interpolation for >) - don't use cache @@ -562,19 +565,52 @@ class CommandAction(ActionBase): str shell script as string or array with tags replaced (direct or as variables). """ - realCmd = cls.replaceTag(realCmd, aInfo, conditional=False) + # array for escaped vars: + varsDict = dict() + + def escapeVal(tag, value): + # if the value should be escaped: + if cls.ESCAPE_CRE.search(value): + # That one needs to be escaped since its content is + # out of our control + tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) + varsDict[tag] = value # add variable + value = '$'+tag # replacement as variable + # replacement for tag: + return value + + # substitution callable, used by interpolation of each tag + def substVal(m): + tag = m.group(1) # tagname from match + try: + value = aInfo[tag] + except KeyError: + # fallback (no or default replacement) + return ADD_REPL_TAGS.get(tag, m.group()) + value = str(value) # assure string + # replacement for tag: + return escapeVal(tag, value) + + # Replace normally properties of aInfo non-recursive: + realCmd = TAG_CRE.sub(substVal, realCmd) + # Replace ticket options (filter capture groups) non-recursive: if '<' in realCmd: tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tn = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.groups()[0]) try: - return str(tickData[tn]) + value = str(tickData[tag]) except KeyError: return "" + return escapeVal("F_"+tag, value) realCmd = FCUSTAG_CRE.sub(substTag, realCmd) + + # build command corresponding "escaped" variables: + if varsDict: + realCmd = Utils.buildShellCmd(realCmd, varsDict) return realCmd def _processCmd(self, cmd, aInfo=None, conditional=''): diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 56428294..a11759b1 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -60,6 +60,7 @@ class Utils(): DEFAULT_SLEEP_TIME = 2 DEFAULT_SLEEP_INTERVAL = 0.2 DEFAULT_SHORT_INTERVAL = 0.001 + DEFAULT_SHORTEST_INTERVAL = DEFAULT_SHORT_INTERVAL / 100 class Cache(object): @@ -183,8 +184,8 @@ class Utils(): def _popen_wait_end(): retcode = popen.poll() return (True, retcode) if retcode is not None else None - # popen.poll is fast operation so we can put down the sleep interval: - retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL / 100) + # popen.poll is fast operation so we can use the shortest sleep interval: + retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORTEST_INTERVAL) if retcode: retcode = retcode[1] # if timeout: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index dc847290..cbd0aaca 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -394,7 +394,7 @@ class CommandActionTest(LogCaptureTestCase): r'''printf %b "foreign input:\n''' r''' -- $f2bV_A --\n''' r''' -- $f2bV_B --\n''' - r''' -- $(echo $f2bV_C) --''' # echo just replaces \n to test it as single line + r''' -- $(echo -n $f2bV_C) --''' # echo just replaces \n to test it as single line r'''"''', varsDict={ 'f2bV_A': 'I\'m a hacker; && $(echo $f2bV_B)', @@ -406,6 +406,34 @@ class CommandActionTest(LogCaptureTestCase): ' -- I"m very bad hacker --', ' -- `Very | very $(bad & worst hacker)` --', all=True) + def testExecuteReplaceEscapeWithVars(self): + self.__action.actionban = 'echo "** ban , reason: ...\\n"' + self.__action.actionunban = 'echo "** unban "' + self.__action.actionstop = 'echo "** stop monitoring"' + matches = [ + '', + '" Hooray! #', + '`I\'m cool script kiddy', + '`I`m very cool > /here-is-the-path/to/bin/.x-attempt.sh', + '', + ] + aInfo = { + 'ip': '192.0.2.1', + 'reason': 'hacking attempt ( he thought he knows how f2b internally works ;)', + 'matches': '\n'.join(matches) + } + self.pruneLog() + self.__action.ban(aInfo) + self.assertLogged( + '** ban %s' % aInfo['ip'], aInfo['reason'], *matches, all=True) + self.assertNotLogged( + '** unban %s' % aInfo['ip'], '** stop monitoring', all=True) + self.pruneLog() + self.__action.unban(aInfo) + self.__action.stop() + self.assertLogged( + '** unban %s' % aInfo['ip'], '** stop monitoring', all=True) + def testExecuteIncorrectCmd(self): CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null') self.assertLogged('HINT on 127: "Command not found"') diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 7a79b6a5..8d1cfa62 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1679,7 +1679,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): # complain -- ('j-complain-abuse', 'complain[' - 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s Hostname: - ",' + + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s \'Hostname: , family: \' - ",' + # test reverse ip: 'debug=1,' + # 2 logs to test grep from multiple logs: @@ -1694,14 +1694,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', # both abuse mails should be separated with space: - 'mail -s Hostname: test-host - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', + 'mail -s Hostname: test-host, family: inet4 - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), 'ip6-ban': ( # test reverse ip: 'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org', 'Lines containing failures of 2001:db8::1 (max 2)', # both abuse mails should be separated with space: - 'mail -s Hostname: test-host - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', + 'mail -s Hostname: test-host, family: inet6 - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), }), ) From 57e9c25449e82d3224207a024d8077513a910ef8 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 17 Mar 2017 22:38:49 +0100 Subject: [PATCH 07/76] bug fix in the config readers: mixing with the init section should affect only own init options (from init section only bypass default section); the situation details: value of "_daemon" from default section "default" (with init section) falsely overwrites it from definition section "test" - the resulting value of "_daemon" should be "test" in all 3 resulting failregex's (as specified in test.local), fixed and covered now; additionally more complex cases covered also (all filter parameters in jail via "%(known/...)s", dynamical interpolation across all, etc); --- fail2ban/client/configparserinc.py | 16 +++++++++-- fail2ban/client/configreader.py | 34 +++++++++++++++-------- fail2ban/client/jailreader.py | 6 ++-- fail2ban/tests/config/filter.d/test.conf | 11 ++++++-- fail2ban/tests/config/filter.d/test.local | 11 +++++++- fail2ban/tests/config/jail.conf | 4 +-- 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 35fa7498..a0e02228 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -32,7 +32,7 @@ from ..helpers import getLogger if sys.version_info >= (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) - from configparser import ConfigParser as SafeConfigParser, \ + from configparser import ConfigParser as SafeConfigParser, NoSectionError, \ BasicInterpolation # And interpolation of __name__ was simply removed, thus we need to @@ -60,7 +60,7 @@ if sys.version_info >= (3,2): parser, option, accum, rest, section, map, depth) else: # pragma: no cover - from ConfigParser import SafeConfigParser + from ConfigParser import SafeConfigParser, NoSectionError # Gets the instance of the logger. logSys = getLogger(__name__) @@ -200,6 +200,18 @@ after = 1.conf def get_sections(self): return self._sections + def options(self, section, onlyOwn=False): + """Return a list of option names for the given section name.""" + try: + opts = self._sections[section] + except KeyError: + raise NoSectionError(section) + if not onlyOwn: + # mix it with defaults: + return set(opts.keys()) | set(self._defaults) + # only own option names: + return opts.keys() + def read(self, filenames, get_includes=True): if not isinstance(filenames, list): filenames = [ filenames ] diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 04502504..6e46e349 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -122,9 +122,9 @@ class ConfigReader(): if self._cfg is not None: return self._cfg.merge_section(*args, **kwargs) - def options(self, *args): + def options(self, section, onlyOwn=False): if self._cfg is not None: - return self._cfg.options(*args) + return self._cfg.options(section, onlyOwn) return {} def get(self, sec, opt, raw=False, vars={}): @@ -297,23 +297,35 @@ class DefinitionInitConfigReader(ConfigReader): self._create_unshared(self._file) return SafeConfigParserWithIncludes.read(self._cfg, self._file) - def getOptions(self, pOpts): + def getOptions(self, pOpts, all=False): # overwrite static definition options with init values, supplied as # direct parameters from jail-config via action[xtra1="...", xtra2=...]: + if not pOpts: + pOpts = dict() if self._initOpts: - if not pOpts: - pOpts = dict() pOpts = _merge_dicts(pOpts, self._initOpts) self._opts = ConfigReader.getOptions( self, "Definition", self._configOpts, pOpts) self._pOpts = pOpts if self.has_section("Init"): - for opt in self.options("Init"): - v = self.get("Init", opt) - if not opt.startswith('known/') and opt != '__name__': + # get only own options (without options from default): + getopt = lambda opt: self.get("Init", opt) + for opt in self.options("Init", onlyOwn=True): + if opt == '__name__': continue + v = None + if not opt.startswith('known/'): + if v is None: v = getopt(opt) self._initOpts['known/'+opt] = v - if not opt in self._initOpts: + if opt not in self._initOpts: + if v is None: v = getopt(opt) self._initOpts[opt] = v + if all and self.has_section("Definition"): + # merge with all definition options (and options from default), + # bypass already converted option (so merge only new options): + for opt in self.options("Definition"): + if opt == '__name__' or opt in self._opts: continue + self._opts[opt] = self.get("Definition", opt) + def _convert_to_boolean(self, value): return value.lower() in ("1", "yes", "true", "on") @@ -336,12 +348,12 @@ class DefinitionInitConfigReader(ConfigReader): def getCombined(self, ignore=()): combinedopts = self._opts - ignore = set(ignore).copy() if self._initOpts: - combinedopts = _merge_dicts(self._opts, self._initOpts) + combinedopts = _merge_dicts(combinedopts, self._initOpts) if not len(combinedopts): return {} # ignore conditional options: + ignore = set(ignore).copy() for n in combinedopts: cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n) if cond: diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 2bef2c4f..58ae2a7f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -139,11 +139,11 @@ class JailReader(ConfigReader): filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() - # merge options from filter as 'known/...': - self.__filter.getOptions(self.__opts) - ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') if not ret: raise JailDefError("Unable to read the filter %r" % filterName) + # merge options from filter as 'known/...' (all options unfiltered): + self.__filter.getOptions(self.__opts, all=True) + ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') else: self.__filter = None logSys.warning("No filter set for jail %s" % self.__name) diff --git a/fail2ban/tests/config/filter.d/test.conf b/fail2ban/tests/config/filter.d/test.conf index f09d3467..9d08ef09 100644 --- a/fail2ban/tests/config/filter.d/test.conf +++ b/fail2ban/tests/config/filter.d/test.conf @@ -1,6 +1,13 @@ #[INCLUDES] #before = common.conf -[Definition] -failregex = failure test 1 (filter.d/test.conf) +[DEFAULT] +_daemon = default +[Definition] +where = conf +failregex = failure <_daemon> (filter.d/test.%(where)s) + +[Init] +# test parameter, should be overriden in jail by "filter=test[one=1,...]" +one = *1* diff --git a/fail2ban/tests/config/filter.d/test.local b/fail2ban/tests/config/filter.d/test.local index 1b6cf55e..a954f81e 100644 --- a/fail2ban/tests/config/filter.d/test.local +++ b/fail2ban/tests/config/filter.d/test.local @@ -2,6 +2,15 @@ #before = common.conf [Definition] +# overwrite default daemon, additionally it should be accessible in jail with "%(known/_daemon)s": +_daemon = test +# interpolate previous regex (from test.conf) + new 2nd + dynamical substitution) of "two" an "where": failregex = %(known/failregex)s - failure test 2 (filter.d/test.local) + failure %(_daemon)s (filter.d/test.) +# parameter "two" should be specified in jail by "filter=test[..., two=2]" +[Init] +# this parameter can be used in jail with "%(known/three)s": +three = 3 +# this parameter "where" does not overwrite "where" in definition of test.conf (dynamical values only): +where = local \ No newline at end of file diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 659e3fd3..64c1b830 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -15,9 +15,9 @@ ignoreip = [test-known-interp] enabled = true -filter = test +filter = test[one=1,two=2] failregex = %(known/failregex)s - failure test 3 (jail.local) + failure %(known/_daemon)s %(known/three)s (jail.local) [missinglogfiles] enabled = true From 32f3c1dbf3fa3c0e3e10dc7b3d8fe864bf67289d Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Mar 2017 13:34:42 +0100 Subject: [PATCH 08/76] test coverage --- fail2ban/client/configreader.py | 39 +++++++++++++++----------- fail2ban/server/utils.py | 2 +- fail2ban/tests/clientreadertestcase.py | 15 ++++++++-- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6e46e349..965a7a37 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -109,33 +109,40 @@ class ConfigReader(): self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) def sections(self): - if self._cfg is not None: + try: return self._cfg.sections() - return [] + except AttributeError: + return [] def has_section(self, sec): - if self._cfg is not None: + try: return self._cfg.has_section(sec) - return False - - def merge_section(self, *args, **kwargs): - if self._cfg is not None: - return self._cfg.merge_section(*args, **kwargs) + except AttributeError: + return False + def merge_section(self, section, *args, **kwargs): + try: + return self._cfg.merge_section(section, *args, **kwargs) + except AttributeError: + raise NoSectionError(section) + def options(self, section, onlyOwn=False): - if self._cfg is not None: + try: return self._cfg.options(section, onlyOwn) - return {} + except AttributeError: + raise NoSectionError(section) def get(self, sec, opt, raw=False, vars={}): - if self._cfg is not None: + try: return self._cfg.get(sec, opt, raw=raw, vars=vars) - return None + except AttributeError: + raise NoSectionError(sec) - def getOptions(self, *args, **kwargs): - if self._cfg is not None: - return self._cfg.getOptions(*args, **kwargs) - return {} + def getOptions(self, section, *args, **kwargs): + try: + return self._cfg.getOptions(section, *args, **kwargs) + except AttributeError: + raise NoSectionError(section) class ConfigReaderUnshared(SafeConfigParserWithIncludes): diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index a11759b1..bb6812c7 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -319,7 +319,7 @@ class Utils(): return e.errno == errno.EPERM else: return True - else: + else: # pragma : no cover (no windows currently supported) @staticmethod def pid_exists(pid): import ctypes diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 37add795..bfa68e03 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -28,7 +28,7 @@ import re import shutil import tempfile import unittest -from ..client.configreader import ConfigReader, ConfigReaderUnshared +from ..client.configreader import ConfigReader, ConfigReaderUnshared, NoSectionError from ..client import configparserinc from ..client.jailreader import JailReader from ..client.filterreader import FilterReader @@ -317,7 +317,17 @@ class JailReaderTest(LogCaptureTestCase): self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) - + def testCommonFunction(self): + c = ConfigReader(share_config={}) + # test common functionalities (no shared, without read of config): + self.assertEqual(c.sections(), []) + self.assertFalse(c.has_section('test')) + self.assertRaises(NoSectionError, c.merge_section, 'test', {}) + self.assertRaises(NoSectionError, c.options, 'test') + self.assertRaises(NoSectionError, c.get, 'test', 'any') + self.assertRaises(NoSectionError, c.getOptions, 'test', {}) + + class FilterReaderTest(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -712,6 +722,7 @@ class JailsReaderTest(LogCaptureTestCase): self.assertEqual(opts['socket'], '/var/run/fail2ban/fail2ban.sock') self.assertEqual(opts['pidfile'], '/var/run/fail2ban/fail2ban.pid') + configurator.readAll() configurator.getOptions() configurator.convertToProtocol() commands = configurator.getConfigStream() From f13fac5ae9a4e219d88d1d01f5d7af46e1ac5e46 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 00:15:57 +0100 Subject: [PATCH 09/76] amend to 5561423be3b2d4636f5484183c3ad470fd326d06: fixed incorrect failure counting despite the `` marked regex; extra: introduced new tag `` as mark to forget current multi-line MLFID (e. g. connection closed); Closes gh-1727 --- config/filter.d/sshd.conf | 14 +++++++------- fail2ban/server/filter.py | 32 ++++++++++++++++++++++---------- fail2ban/tests/files/logs/sshd | 5 +++++ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 320ab59c..95915fcc 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -37,24 +37,24 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ ^User .+ from not allowed because not in any group\s*%(__suff)s$ ^refused connect from \S+ \(\)\s*%(__suff)s$ - ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ + ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s - ^Received disconnect from : 11: - ^Connection closed by %(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s + ^Received disconnect from : 11: + ^Connection closed by %(__suff)s$ mdre-normal = mdre-ddos = ^Did not receive identification string from %(__suff)s$ - ^Connection reset by %(__on_port_opt)s%(__suff)s + ^Connection reset by %(__on_port_opt)s%(__suff)s ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: - ^Read from socket failed: Connection reset by peer%(__suff)s + ^Read from socket failed: Connection reset by peer%(__suff)s -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$ diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ca2dae86..cbdb4857 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -554,20 +554,29 @@ class Filter(JailThread): mlfidGroups = mlfidFail[1] # if current line not failure, but previous was failure: if fail.get('nofail') and not mlfidGroups.get('nofail'): - del fail['nofail'] # remove nofail flag - was already market as failure + del fail['nofail'] # remove nofail flag - completed with fid (host, ip) self.mlfidCache.unset(mlfid) # remove cache entry # if current line is failure, but previous was not: elif not fail.get('nofail') and mlfidGroups.get('nofail'): - del mlfidGroups['nofail'] # remove nofail flag + del mlfidGroups['nofail'] # remove nofail flag - completed as failure self.mlfidCache.unset(mlfid) # remove cache entry + else: + # cache this line info (if not forget): + if not fail.get('mlfforget'): + mlfidFail = [self.__lastDate, fail] + self.mlfidCache.set(mlfid, mlfidFail) + else: + self.mlfidCache.unset(mlfid) # remove cache entry + return fail fail2 = mlfidGroups.copy() fail2.update(fail) fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() fail = fail2 - elif fail.get('nofail'): - fail["matches"] = failRegex.getMatchedTupleLines() + elif not fail.get('mlfforget'): mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) + if fail.get('nofail'): + fail["matches"] = failRegex.getMatchedTupleLines() return fail @@ -683,6 +692,11 @@ class Filter(JailThread): mlfid = fail.get('mlfid') if mlfid is not None: fail = self._mergeFailure(mlfid, fail, failRegex) + # bypass if no-failure case: + if fail.get('nofail'): + logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure", + mlfid, failRegexIndex) + if not self.checkAllRegex: return failList else: # matched lines: fail["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() @@ -702,18 +716,16 @@ class Filter(JailThread): host = fail.get('dns') if host is None: # first try to check we have mlfid case (cache connection id): - if fid is None: - if mlfid: - fail = self._mergeFailure(mlfid, fail, failRegex) - else: + if fid is None and mlfid is None: # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: fid = failRegex.getFailID() host = fid cidr = IPAddr.CIDR_RAW # if mlfid case (not failure): if host is None: - if not self.checkAllRegex: # or fail.get('nofail'): - return failList + logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier", + mlfid, failRegexIndex) + if not self.checkAllRegex: return failList ips = [None] # if raw - add single ip or failure-id, # otherwise expand host to multiple ips using dns (or ignore it if not valid): diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index fe19591c..6f9a1468 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -113,6 +113,11 @@ May 27 00:16:33 host sshd[2364]: Received disconnect from 198.51.100.76: 11: Bye # failJSON: { "time": "2004-09-29T16:28:02", "match": true , "host": "127.0.0.1" } Sep 29 16:28:02 spaceman sshd[16699]: Failed password for dan from 127.0.0.1 port 45416 ssh1 +# failJSON: { "match": false, "desc": "no failure, just cache mlfid (conn-id)" } +Sep 29 16:28:05 localhost sshd[16700]: Connection from 192.0.2.5 +# failJSON: { "match": false, "desc": "no failure, just covering mlfid (conn-id) forget" } +Sep 29 16:28:05 localhost sshd[16700]: Connection closed by 192.0.2.5 [preauth] + # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: RSA 8c:e3:aa:0f:64:51:02:f7:14:79:89:3f:65:84:7c:30, client user "dan", client host "localhost.localdomain" From 1971fd4bd3eda0bfe91ee688ee41906af93043fc Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 00:30:40 +0100 Subject: [PATCH 10/76] don't remove MLFID from cache (can recognize multiple attempt within the same connection) --- fail2ban/client/jailreader.py | 4 ++-- fail2ban/server/filter.py | 26 +++++++++++--------------- fail2ban/tests/files/logs/sshd | 4 +++- fail2ban/tests/samplestestcase.py | 6 +++--- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 2bef2c4f..bdb3564f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -220,8 +220,8 @@ class JailReader(ConfigReader): if self.__filter: stream.extend(self.__filter.convert()) for opt, value in self.__opts.iteritems(): - if opt == "logpath" and \ - not self.__opts.get('backend', None).startswith("systemd"): + if opt == "logpath": + if self.__opts.get('backend', None).startswith("systemd"): continue found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cbdb4857..76787cc7 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -550,26 +550,22 @@ class Filter(JailThread): def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None + # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] - # if current line not failure, but previous was failure: - if fail.get('nofail') and not mlfidGroups.get('nofail'): - del fail['nofail'] # remove nofail flag - completed with fid (host, ip) - self.mlfidCache.unset(mlfid) # remove cache entry - # if current line is failure, but previous was not: - elif not fail.get('nofail') and mlfidGroups.get('nofail'): - del mlfidGroups['nofail'] # remove nofail flag - completed as failure - self.mlfidCache.unset(mlfid) # remove cache entry + # update - if not forget (disconnect/reset): + if not fail.get('mlfforget'): + mlfidGroups.update(fail) else: - # cache this line info (if not forget): - if not fail.get('mlfforget'): - mlfidFail = [self.__lastDate, fail] - self.mlfidCache.set(mlfid, mlfidFail) - else: - self.mlfidCache.unset(mlfid) # remove cache entry - return fail + self.mlfidCache.unset(mlfid) # remove cached entry + # merge with previous info: fail2 = mlfidGroups.copy() fail2.update(fail) + if not fail.get('nofail'): # be sure we've correct current state + try: + del fail2['nofail'] + except KeyError: + pass fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() fail = fail2 elif not fail.get('mlfforget'): diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 6f9a1468..f465c9a7 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -238,4 +238,6 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 # failJSON: { "match": false } Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } -Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file +Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" } +Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 00a8f305..5c45e729 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -200,13 +200,13 @@ def testSampleRegexsFactory(name, basedir): self.assertEqual(len(ret), 1, "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) - # Fallback for backwards compatibility (previously no fid, was host only): - if faildata.get("host", None) is not None and fail.get("host", None) is None: - fail["host"] = fid # Verify match captures (at least fid/host) and timestamp as expected for k, v in faildata.iteritems(): if k not in ("time", "match", "desc"): fv = fail.get(k, None) + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host" and fv is None: + fv = fid self.assertEqual(fv, v) t = faildata.get("time", None) From b6886f2e519829dbf6124eb64e7887f3b6d6f171 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 09:42:27 +0100 Subject: [PATCH 11/76] SampleRegexsFactory extended with optional filter constraint, if testing the same log-file with multiple filters (no possibility to match by the old sshd-filter 'zzz-sshd-obsolete-multiline') --- fail2ban/tests/files/logs/sshd | 4 ++-- fail2ban/tests/samplestestcase.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index f465c9a7..fb3defea 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -239,5 +239,5 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" } -Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 5c45e729..0ba11c2e 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -182,6 +182,9 @@ def testSampleRegexsFactory(name, basedir): try: ret = self.filter.processLine(line) if not ret: + # Bypass if filter constraint specified: + if faildata.get('filter') and name != faildata.get('filter'): + continue # Check line is flagged as none match self.assertFalse(faildata.get('match', True), "Line not matched when should have") @@ -202,7 +205,7 @@ def testSampleRegexsFactory(name, basedir): # Verify match captures (at least fid/host) and timestamp as expected for k, v in faildata.iteritems(): - if k not in ("time", "match", "desc"): + if k not in ("time", "match", "desc", "filter"): fv = fail.get(k, None) # Fallback for backwards compatibility (previously no fid, was host only): if k == "host" and fv is None: From 43d2cae8dae53eb51d3841450de6b0a03a2e1218 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 10:17:16 +0100 Subject: [PATCH 12/76] small amend that correct log trace output by forget MLFID (outputs the reason why it was forgotten - close, disconnect, etc.) --- fail2ban/server/filter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 76787cc7..72bf47d8 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -690,8 +690,8 @@ class Filter(JailThread): fail = self._mergeFailure(mlfid, fail, failRegex) # bypass if no-failure case: if fail.get('nofail'): - logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure", - mlfid, failRegexIndex) + logSys.log(7, "Nofail by mlfid %r in regex %s: %s", + mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure")) if not self.checkAllRegex: return failList else: # matched lines: @@ -719,8 +719,8 @@ class Filter(JailThread): cidr = IPAddr.CIDR_RAW # if mlfid case (not failure): if host is None: - logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier", - mlfid, failRegexIndex) + logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", + mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) if not self.checkAllRegex: return failList ips = [None] # if raw - add single ip or failure-id, From 7a03c964c29fe3f13c3993d98ff77b6e958885f5 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 21 Mar 2017 14:04:18 +0100 Subject: [PATCH 13/76] Update ChangeLog --- ChangeLog | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 5c1ba70e..7629fb2f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,9 +41,13 @@ TODO: implementing of options resp. other tasks from PR #1346 using single-line expressions: - tag ``: used to identify resp. store failure info for groups of log-lines with the same identifier (e. g. combined failure-info for the same conn-id by `(?:conn-id)`, - see sshd.conf for example) + see sshd.conf for example); + - tag ``: can be used as mark to forget current multi-line MLFID (e. g. by connection + closed, reset or disconnect etc); - tag ``: used as mark for no-failure (helper to accumulate common failure-info, e. g. from lines that contain IP-address); + Opposite to obsolete multi-line parsing (using buffering with `maxlines`) it is more precise and + can recognize multiple failure attempts within the same connection (MLFID). * Several filters optimized with pre-filtering using new option `prefregex`, and multiline filter using `` + `` combination; * Exposes filter group captures in actions (non-recursive interpolation of tags ``, From 6ba0546824a93bf5cbe445a6ca575306c1ddddc8 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 14:53:33 +0100 Subject: [PATCH 14/76] code review and inline docu --- fail2ban/client/configparserinc.py | 9 ++++++--- fail2ban/client/configreader.py | 10 +++++++--- fail2ban/server/utils.py | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index a0e02228..6de513cd 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -200,13 +200,16 @@ after = 1.conf def get_sections(self): return self._sections - def options(self, section, onlyOwn=False): - """Return a list of option names for the given section name.""" + def options(self, section, withDefault=True): + """Return a list of option names for the given section name. + + Parameter `withDefault` controls the include of names from section `[DEFAULT]` + """ try: opts = self._sections[section] except KeyError: raise NoSectionError(section) - if not onlyOwn: + if withDefault: # mix it with defaults: return set(opts.keys()) | set(self._defaults) # only own option names: diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 965a7a37..bbc18384 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -126,9 +126,13 @@ class ConfigReader(): except AttributeError: raise NoSectionError(section) - def options(self, section, onlyOwn=False): + def options(self, section, withDefault=False): + """Return a list of option names for the given section name. + + Parameter `withDefault` controls the include of names from section `[DEFAULT]` + """ try: - return self._cfg.options(section, onlyOwn) + return self._cfg.options(section, withDefault) except AttributeError: raise NoSectionError(section) @@ -317,7 +321,7 @@ class DefinitionInitConfigReader(ConfigReader): if self.has_section("Init"): # get only own options (without options from default): getopt = lambda opt: self.get("Init", opt) - for opt in self.options("Init", onlyOwn=True): + for opt in self.options("Init", withDefault=False): if opt == '__name__': continue v = None if not opt.startswith('known/'): diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index bb6812c7..58363e76 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -118,6 +118,15 @@ class Utils(): @staticmethod def buildShellCmd(realCmd, varsDict): + """Generates new shell command as array, contains map as variables to + arguments statement (varsStat), the command (realCmd) used this variables and + the list of the arguments, mapped from varsDict + + Example: + buildShellCmd('echo "V2: $v2, V1: $v1"', {"v1": "val 1", "v2": "val 2", "vUnused": "unused var"}) + returns: + ['v1=$0 v2=$1 vUnused=$2 \necho "V2: $v2, V1: $v1"', 'val 1', 'val 2', 'unused var'] + """ # build map as array of vars and command line array: varsStat = "" if not isinstance(realCmd, list): @@ -172,7 +181,7 @@ class Utils(): else: # pragma: no cover - currently unused env = _merge_dicts(os.environ, varsDict) realCmdId = id(realCmd) - outCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd) + logCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd) try: popen = subprocess.Popen( realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env, @@ -190,7 +199,7 @@ class Utils(): retcode = retcode[1] # if timeout: if retcode is None: - if outCmd: outCmd(logging.ERROR); outCmd = None + if logCmd: logCmd(logging.ERROR); logCmd = None logSys.error("%x -- timed out after %s seconds." % (realCmdId, timeout)) pgid = os.getpgid(popen.pid) @@ -207,6 +216,7 @@ class Utils(): if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover retcode = signal.SIGKILL except OSError as e: + if logCmd: logCmd(logging.ERROR); logCmd = None stderr = "%s -- failed with %s" % (realCmd, e) logSys.error(stderr) if not popen: @@ -214,7 +224,7 @@ class Utils(): std_level = logging.DEBUG if retcode in success_codes else logging.ERROR if std_level > logSys.getEffectiveLevel(): - if outCmd: outCmd(std_level-1); outCmd = None + if logCmd: logCmd(std_level-1); logCmd = None # if we need output (to return or to log it): if output or std_level >= logSys.getEffectiveLevel(): From 5e93bf9bd3ec66707cb961c851a0ce448be3a02b Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Mar 2017 15:52:31 +0100 Subject: [PATCH 15/76] Introduced new option "ignoreself", specifies whether the local resp. own IP addresses should be ignored (default is true). Fail2ban will not ban a host which matches such addresses. Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS resp. IPs of the host self. --- config/jail.conf | 12 +++++++---- fail2ban/client/jailreader.py | 1 + fail2ban/protocol.py | 2 ++ fail2ban/server/filter.py | 18 ++++++++++++++++ fail2ban/server/ipdns.py | 36 ++++++++++++++++++++++++++++++++ fail2ban/server/server.py | 6 ++++++ fail2ban/server/transmitter.py | 6 ++++++ fail2ban/tests/filtertestcase.py | 11 ++++++++++ fail2ban/tests/servertestcase.py | 10 +++++++++ 9 files changed, 98 insertions(+), 4 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c5440b71..7e5cc9b7 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -44,10 +44,14 @@ before = paths-debian.conf # MISCELLANEOUS OPTIONS # -# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not -# ban a host which matches an address in this list. Several addresses can be -# defined using space (and/or comma) separator. -ignoreip = 127.0.0.1/8 ::1 +# "ignorself" specifies whether the local resp. own IP addresses should be ignored +# (default is true). Fail2ban will not ban a host which matches such addresses. +#ignorself = true + +# "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban +# will not ban a host which matches an address in this list. Several addresses +# can be defined using space (and/or comma) separator. +#ignoreip = 127.0.0.1/8 ::1 # External command that will take an tagged arguments to ignore, e.g. , # and return true if the IP is to be ignored. False otherwise. diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 7f69155f..ca092990 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -110,6 +110,7 @@ class JailReader(ConfigReader): ["string", "failregex", None], ["string", "ignoreregex", None], ["string", "ignorecommand", None], + ["bool", "ignoreself", None], ["string", "ignoreip", None], ["string", "filter", ""], ["string", "datepattern", None], diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index d1c33d88..3625ec01 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -81,6 +81,7 @@ protocol = [ ["status [FLAVOR]", "gets the current status of , with optional flavor or extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], +["set ignoreself true|false", "allows the ignoring of own IP addresses"], ["set addignoreip ", "adds to the ignore list of "], ["set delignoreip ", "removes from the ignore list of "], ["set addlogpath ['tail']", "adds to the monitoring list of , optionally starting at the 'tail' of the file (default 'head')."], @@ -117,6 +118,7 @@ protocol = [ ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], +["get ignoreself", "gets the current value of the ignoring the own IP addresses"], ["get ignoreip", "gets the list of ignored IP addresses for "], ["get ignorecommand", "gets ignorecommand of "], ["get failregex", "gets the list of regular expressions which matches the failures for "], diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 72bf47d8..b425060a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -76,6 +76,8 @@ class Filter(JailThread): self.setUseDns(useDns) ## The amount of time to look back. self.__findTime = 600 + ## Ignore own IPs flag: + self.__ignoreSelf = True ## The ignore IP list. self.__ignoreIpList = [] ## Size of line buffer @@ -413,6 +415,17 @@ class Filter(JailThread): return ip + ## + # Ignore own IP/DNS. + # + @property + def ignoreSelf(self): + return self.__ignoreSelf + + @ignoreSelf.setter + def ignoreSelf(self, value): + self.__ignoreSelf = value + ## # Add an IP/DNS to the ignore list. # @@ -458,6 +471,11 @@ class Filter(JailThread): def inIgnoreIPList(self, ip, log_ignore=False): if not isinstance(ip, IPAddr): ip = IPAddr(ip) + + # check own IPs should be ignored and 'ip' is self IP: + if self.__ignoreSelf and ip in DNSUtils.getSelfIPs(): + return True + for net in self.__ignoreIpList: # check if the IP is covered by ignore IP if ip.isInNet(net): diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index bd3b812d..bda32ae8 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -118,6 +118,42 @@ class DNSUtils: return ipList + @staticmethod + def getSelfNames(): + """Get own host names of self""" + # try find cached own hostnames (this tuple-key cannot be used elsewhere): + key = ('self','dns') + names = DNSUtils.CACHE_ipToName.get(key) + # get it using different ways (a set with names of localhost, hostname, fully qualified): + if names is None: + names = set(['localhost']) + for hostname in (socket.gethostname, socket.getfqdn): + try: + names |= set([hostname()]) + except Exception as e: # pragma: no cover + logSys.warning("Retrieving own hostnames failed: %s", e) + # cache and return : + DNSUtils.CACHE_ipToName.set(key, names) + return names + + @staticmethod + def getSelfIPs(): + """Get own IP addresses of self""" + # try find cached own IPs (this tuple-key cannot be used elsewhere): + key = ('self','ips') + ips = DNSUtils.CACHE_nameToIp.get(key) + # get it using different ways (a set with IPs of localhost, hostname, fully qualified): + if ips is None: + ips = set() + for hostname in DNSUtils.getSelfNames(): + try: + ips |= set(DNSUtils.textToIp(hostname, 'yes')) + except Exception as e: # pragma: no cover + logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) + # cache and return : + DNSUtils.CACHE_nameToIp.set(key, ips) + return ips + ## # Class for IP address handling. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index dfab1e38..facbe393 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -308,6 +308,12 @@ class Server: return self.__jails[name].idle # Filter + def setIgnoreSelf(self, name, value): + self.__jails[name].filter.ignoreSelf = value + + def getIgnoreSelf(self, name): + return self.__jails[name].filter.ignoreSelf + def addIgnoreIP(self, name, ip): self.__jails[name].filter.addIgnoreIP(ip) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index ad21b851..bc9edd43 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -181,6 +181,10 @@ class Transmitter: raise Exception("Invalid idle option, must be 'on' or 'off'") return self.__server.getIdleJail(name) # Filter + elif command[1] == "ignoreself": + value = command[2] + self.__server.setIgnoreSelf(name, value) + return self.__server.getIgnoreSelf(name) elif command[1] == "addignoreip": value = command[2] self.__server.addIgnoreIP(name, value) @@ -341,6 +345,8 @@ class Transmitter: return self.__server.getLogEncoding(name) elif command[1] == "journalmatch": # pragma: systemd no cover return self.__server.getJournalMatch(name) + elif command[1] == "ignoreself": + return self.__server.getIgnoreSelf(name) elif command[1] == "ignoreip": return self.__server.getIgnoreIP(name) elif command[1] == "ignorecommand": diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index d3217555..10310a5d 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -325,6 +325,17 @@ class IgnoreIP(LogCaptureTestCase): LogCaptureTestCase.setUp(self) self.jail = DummyJail() self.filter = FileFilter(self.jail) + self.filter.ignoreSelf = False + + def testIgnoreSelfIP(self): + ipList = ("127.0.0.1",) + # test ignoreSelf is false: + for ip in ipList: + self.assertFalse(self.filter.inIgnoreIPList(ip)) + # test ignoreSelf with true: + self.filter.ignoreSelf = True + for ip in ipList: + self.assertTrue(self.filter.inIgnoreIPList(ip)) def testIgnoreIPOK(self): ipList = "127.0.0.1", "192.168.0.1", "255.255.255.255", "99.99.99.99" diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 8d1cfa62..51ff8880 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -449,6 +449,16 @@ class Transmitter(TransmitterBase): self.transm.proceed(["set", self.jailName, "delignoreip", value]), (0, [value])) + self.assertEqual( + self.transm.proceed(["get", self.jailName, "ignoreself"]), + (0, True)) + self.assertEqual( + self.transm.proceed(["set", self.jailName, "ignoreself", False]), + (0, False)) + self.assertEqual( + self.transm.proceed(["get", self.jailName, "ignoreself"]), + (0, False)) + def testJailIgnoreCommand(self): self.setGetTest("ignorecommand", "bin ", jail=self.jailName) From 6c4b1c720475613b4765d040b4d86857a4ccf52b Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Mar 2017 15:54:53 +0100 Subject: [PATCH 16/76] Update ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7629fb2f..b6e7bacd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -68,6 +68,10 @@ TODO: implementing of options resp. other tasks from PR #1346 * Samples test case factory extended with filter options - dict in JSON to control filter options (e. g. mode, etc.): # filterOptions: {"mode": "aggressive"} +* Introduced new jail option "ignoreself", specifies whether the local resp. own IP addresses + should be ignored (default is true). Fail2ban will not ban a host which matches such addresses. + Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS + resp. IPs of the host self. ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc From 663bc9903d69e95e19b67a0bf77c462a4f5fb5ad Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Mar 2017 16:19:21 +0100 Subject: [PATCH 17/76] increase coverage (was decreased since "ignoreip" was set to default empty) --- fail2ban/tests/fail2banclienttestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 683368ee..35e00421 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -780,6 +780,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "findtime = 10m", "failregex = ^\s*failure 401|403 from ", "datepattern = {^LN-BEG}EPOCH", + "ignoreip = 127.0.0.1/8 ::1", # just to cover ignoreip in jailreader/transmitter "", "[test-jail1]", "backend = " + backend, "filter =", "action = ", From 30352c5f03263600e809fb3fa857cd99df1ba40d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Mar 2017 17:48:52 +0100 Subject: [PATCH 18/76] fix sporadic coverage changes (sometimes produces "no such process" in popen.poll after terminate/kill in timeout test cases) --- fail2ban/server/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 58363e76..58363ff0 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -211,7 +211,8 @@ class Utils(): if retcode is None or tout_kill_tree: # Still going... os.killpg(pgid, signal.SIGKILL) # Kill the process time.sleep(Utils.DEFAULT_SLEEP_INTERVAL) - retcode = popen.poll() + if retcode is None: # pragma: no cover - too sporadic + retcode = popen.poll() #logSys.debug("%s -- killed %s ", realCmd, retcode) if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover retcode = signal.SIGKILL From e7052e9625926370365de5cef02e08e8ac548e3b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Mar 2017 09:55:20 +0100 Subject: [PATCH 19/76] update man/jail.conf.5 (docu for the ignoreself) --- man/jail.conf.5 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 2e333e5a..5a75369c 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -199,11 +199,14 @@ Arguments can be passed to actions to override the default values from the [Init Values can also be quoted (required when value includes a ","). More that one action can be specified (in separate lines). .RE .TP +.B ignoreself +boolean value (default true) indicates the banning of own IP addresses should be prevented +.TP .B ignoreip -list of IPs not to ban. They can include a CIDR mask too. +list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. .TP .B ignorecommand -command that is executed to determine if the current candidate IP for banning should not be banned. +command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit. .br IP will not be banned if command returns successfully (exit code 0). Like ACTION FILES, tags like are can be included in the ignorecommand value and will be substituted before execution. Currently only is supported however more will be added later. From 61c1bdfe79a98be3f1aa7ccb2957bf2299d768fc Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Mar 2017 22:02:37 +0100 Subject: [PATCH 20/76] Normalizes replacement of `` (moved to _resolveHostTag, so will be replaced together with another tags); Regex will be compiled as MULTILINE only if needed (buffering with `maxlines` > 1), that enables: - improve performance by the single line parsing; - make regex more precise (because distinguish between anchors `^`/`$` for the begin/end of string and the new-line character '\n', e. g. if coming from filters (like systemd journal) that allow the parsing of log-entries contain new-line chars (as single entry); --- fail2ban/server/failregex.py | 26 ++++++++++++++++++-------- fail2ban/server/filter.py | 8 ++++++-- fail2ban/tests/samplestestcase.py | 2 +- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 59f59978..c1e7107a 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -103,20 +103,17 @@ class Regex: # avoid construction of invalid object. # @param value the regular expression - def __init__(self, regex, **kwargs): + def __init__(self, regex, multiline=False, **kwargs): self._matchCache = None # Perform shortcuts expansions. - # Resolve "" tag using default regular expression for host: + # Replace standard f2b-tags (like "", etc) using default regular expressions: regex = Regex._resolveHostTag(regex, **kwargs) - # Replace "" with regular expression for multiple lines. - regexSplit = regex.split("") - regex = regexSplit[0] - for n, regexLine in enumerate(regexSplit[1:]): - regex += "\n(?P(?:(.*\n)*?))" % n + regexLine + # if regex.lstrip() == '': raise RegexException("Cannot add empty regex") + flags = re.MULTILINE if (multiline or "\n" in regex or r"\n" in regex) else 0 try: - self._regexObj = re.compile(regex, re.MULTILINE) + self._regexObj = re.compile(regex, flags) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % @@ -125,6 +122,11 @@ class Regex: def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) + @property + def flags(self): + """Returns the regex matching flags combination of the compiled regex object""" + return self._regexObj.flags + ## # Replaces "", "", "", "" with default regular expression for host # @@ -135,6 +137,9 @@ class Regex: def _resolveHostTag(regex, useDns="yes"): openTags = dict() + props = { + 'nl': 0, # new lines counter by tag; + } # tag interpolation callable: def substTag(m): tag = m.group() @@ -142,6 +147,11 @@ class Regex: # 3 groups instead of - separated ipv4, ipv6 and host (dns) if tn == "HOST": return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR] + # replace "" with regular expression for multiple lines (by buffering with maxlines) + if tn == "SKIPLINES": + nl = props['nl'] + props['nl'] = nl + 1 + return r"\n(?P(?:(?:.*\n)*?))" % (nl,) # static replacement from RH4TAG: try: return RH4TAG[tn] diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b425060a..ed50e8c1 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -161,10 +161,14 @@ class Filter(JailThread): # @param value the regular expression def addFailRegex(self, value): + multiLine = self.getMaxLines() > 1 try: - regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns) + regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, + useDns=self.__useDns) self.__failRegex.append(regex) - if "\n" in regex.getRegex() and not self.getMaxLines() > 1: + regexExpr = regex.getRegex() + # check new lines present in regex (was compiled as multiline), incorrect by `maxlines=1`: + if (regex.flags & re.MULTILINE) and not multiLine: logSys.warning( "Mutliline regex set for jail %r " "but maxlines not greater than 1", self.jailName) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 0ba11c2e..fb6812cb 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -41,7 +41,7 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") # regexp to test greedy catch-all should be not-greedy: RE_HOST = Regex('').getRegex() -RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?).*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$') +RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$') class FilterSamplesRegex(unittest.TestCase): From bc888e07533bcebd81a9fedf7ce846d86851d63b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Mar 2017 12:05:51 +0100 Subject: [PATCH 21/76] Regex compiled in multi-line parsing mode only if `maxlines` > 1 (buffering), if however expected - prefix `(?m)` could be used in regex to enable it; Removed warning "Mutliline regex set for jail ... but maxlines not greater than 1", because can be expected situation now: non multi-line entry from systemd-filter containing new-lines (that should be ignored by anchors resp. entry parsed as single string); small code review; --- fail2ban/server/failregex.py | 8 +------- fail2ban/server/filter.py | 6 ------ fail2ban/tests/filtertestcase.py | 6 +++--- fail2ban/tests/samplestestcase.py | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index c1e7107a..d5c9345f 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -111,9 +111,8 @@ class Regex: # if regex.lstrip() == '': raise RegexException("Cannot add empty regex") - flags = re.MULTILINE if (multiline or "\n" in regex or r"\n" in regex) else 0 try: - self._regexObj = re.compile(regex, flags) + self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % @@ -122,11 +121,6 @@ class Regex: def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) - @property - def flags(self): - """Returns the regex matching flags combination of the compiled regex object""" - return self._regexObj.flags - ## # Replaces "", "", "", "" with default regular expression for host # diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ed50e8c1..75536d57 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -166,12 +166,6 @@ class Filter(JailThread): regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, useDns=self.__useDns) self.__failRegex.append(regex) - regexExpr = regex.getRegex() - # check new lines present in regex (was compiled as multiline), incorrect by `maxlines=1`: - if (regex.flags & re.MULTILINE) and not multiLine: - logSys.warning( - "Mutliline regex set for jail %r " - "but maxlines not greater than 1", self.jailName) except RegexException as e: logSys.error(e) raise e diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 10310a5d..ce665e72 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1479,8 +1479,8 @@ class GetFailures(LogCaptureTestCase): output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) - self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) @@ -1497,9 +1497,9 @@ class GetFailures(LogCaptureTestCase): def testGetFailuresMultiLineIgnoreRegex(self): output = [("192.0.43.10", 2, 1124013599.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) + self.filter.setMaxLines(100) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") - self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) @@ -1513,9 +1513,9 @@ class GetFailures(LogCaptureTestCase): ("192.0.43.11", 1, 1124013598.0), ("192.0.43.15", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) + self.filter.setMaxLines(100) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") - self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index fb6812cb..121c1c5c 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -40,7 +40,7 @@ TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config") TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") # regexp to test greedy catch-all should be not-greedy: -RE_HOST = Regex('').getRegex() +RE_HOST = Regex._resolveHostTag('') RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$') From 990d9a66da60ceca5c1acca83ba425db8c78b726 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Mar 2017 17:07:21 +0100 Subject: [PATCH 22/76] fail2ban-regex: fixed matched output by multi-line (buffered) parsing + and multi-line debuggex URL; test coverage extended; --- fail2ban/client/fail2banregex.py | 47 +++++++++++++++++++------ fail2ban/tests/fail2banregextestcase.py | 37 +++++++++++++++++++ 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 782513a7..45dbfe95 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -55,11 +55,14 @@ from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, # Gets the instance of the logger. logSys = getLogger("fail2ban") -def debuggexURL(sample, regex, useDns="yes"): - q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns), - 'str': sample, - 'flavor': 'python' }) - return 'https://www.debuggex.com/?' + q +def debuggexURL(sample, regex, multiline=False, useDns="yes"): + args = { + 're': Regex._resolveHostTag(regex, useDns=useDns), + 'str': sample, + 'flavor': 'python' + } + if multiline: args['flags'] = 'm' + return 'https://www.debuggex.com/?' + urllib.urlencode(args) def output(args): # pragma: no cover (overriden in test-cases) print(args) @@ -400,6 +403,7 @@ class Fail2banRegex(object): fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() try: ret = self._filter.processLine(line, date) + lines = [] line = self._filter.processedLine() for match in ret: # Append True/False flag depending if line was matched by @@ -422,9 +426,17 @@ class Fail2banRegex(object): "".join(bufLine[::2]))) except ValueError: pass - else: - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 + if lines: # pre-lines parsed in multiline mode (buffering) + lines.append(line) + line = "\n".join(lines) return line, ret def process(self, test_lines): @@ -472,6 +484,7 @@ class Fail2banRegex(object): assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] + multiline = self._filter.getMaxLines() > 1 if lines: header = "%s line(s):" % (ltype.capitalize(),) if self._debuggex: @@ -485,7 +498,8 @@ class Fail2banRegex(object): for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + - debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans) + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), + multiline, self._opts.usedns), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ @@ -599,8 +613,19 @@ class Fail2banRegex(object): output( "Use journal match : %s" % " ".join(journalmatch) ) test_lines = journal_lines_gen(flt, myjournal) else: - output( "Use single line : %s" % shortstr(cmd_log) ) - test_lines = [ cmd_log ] + # if single line parsing (without buffering) + if self._filter.getMaxLines() <= 1: + output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) + test_lines = [ cmd_log ] + else: # multi line parsing (with buffering) + test_lines = cmd_log.split("\n") + output( "Use multi line : %s line(s)" % len(test_lines) ) + for i, l in enumerate(test_lines): + if i >= 5: + output( "| ..." ); break + output( "| %2.2s: %s" % (i+1, shortstr(l)) ) + output( "`-" ) + output( "" ) self.process(test_lines) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 0cd0e303..d865b34d 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -252,6 +252,43 @@ class Fail2banRegexTest(LogCaptureTestCase): ) self.assertTrue(fail2banRegex.start(args)) + def testDirectMultilineBuf(self): + # test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed): + for preLines in (0, 20): + self.pruneLog("[test-phase %s]" % preLines) + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5", + ("1490349000 TEST-NL\n"*preLines) + + "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", + r"^\s*FAIL\s*$^\s*HOST \s*$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2)) + # both matched lines were printed: + self.assertLogged("| 1490349000 FAIL", "| 1490349001 HOST 192.0.2.34", all=True) + + + def testDirectMultilineBufDebuggex(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5", + "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", + r"^\s*FAIL\s*$^\s*HOST \s*$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + self.assertLogged("&flags=m") + + def testSinglelineWithNLinContent(self): + # + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--print-all-matched", + "1490349000 FAIL: failure\nhost: 192.0.2.35", + r"^\s*FAIL:\s*.*\nhost:\s+$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + + def testWrongFilterFile(self): # use test log as filter file to cover eror cases... (opts, args, fail2banRegex) = _Fail2banRegex( From 6ac5c55edcf7548cb52beffab096d717cb5e46bd Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Mar 2017 17:35:41 +0100 Subject: [PATCH 23/76] the sequence in args-dict is currently undefined (so can be 1st argument with `?` instead of `&`) --- fail2ban/tests/fail2banregextestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index d865b34d..8bfedad1 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -276,7 +276,8 @@ class Fail2banRegexTest(LogCaptureTestCase): ) self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') - self.assertLogged("&flags=m") + # the sequence in args-dict is currently undefined (so can be 1st argument) + self.assertLogged("&flags=m", "?flags=m") def testSinglelineWithNLinContent(self): # From 52c19503715ac5e21aa98e5cc6bfcfd36c9acda3 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 24 Mar 2017 19:03:17 +0100 Subject: [PATCH 24/76] Update mysqld-auth.conf small typo, closes gh-1725 (Thx @seth-reeser) --- config/filter.d/mysqld-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 3ad70cb7..31bd2056 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -1,4 +1,4 @@ -# Fail2Ban filter for unsuccesfull MySQL authentication attempts +# Fail2Ban filter for unsuccesful MySQL authentication attempts # # # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]: From c82495353fdc04b7b7a9ee8d63ef117ba4a1a9c9 Mon Sep 17 00:00:00 2001 From: Seth Reeser Date: Fri, 24 Mar 2017 14:03:20 -0400 Subject: [PATCH 25/76] Update mysqld-auth.conf (#1725) --- config/filter.d/mysqld-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 3ad70cb7..31bd2056 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -1,4 +1,4 @@ -# Fail2Ban filter for unsuccesfull MySQL authentication attempts +# Fail2Ban filter for unsuccesful MySQL authentication attempts # # # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]: From d26060ead0a2b241c495bc8d777ab3e2cc9e788e Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 27 Mar 2017 09:38:53 +0200 Subject: [PATCH 26/76] Update ChangeLog belongs to #1733 --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index b6e7bacd..eafaef0f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,8 @@ TODO: implementing of options resp. other tasks from PR #1346 - fixed using new tag `` (sh/dash compliant now) * `action.d/sendmail-geoip-lines.conf` - fixed using new tag `` (without external command execution) +* fail2ban-regex: fixed matched output by multi-line (buffered) parsing +* fail2ban-regex: support for multi-line debuggex URL implemented (gh-422) ### New Features * New Actions: @@ -72,6 +74,13 @@ TODO: implementing of options resp. other tasks from PR #1346 should be ignored (default is true). Fail2ban will not ban a host which matches such addresses. Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS resp. IPs of the host self. +* Regex will be compiled as MULTILINE only if needed (buffering with `maxlines` > 1), that enables: + - to improve performance by the single line parsing (see gh-1733); + - make regex more precise (because distinguish between anchors `^`/`$` for the begin/end of string + and the new-line character '\n', e. g. if coming from filters (like systemd journal) that allow + the parsing of log-entries contain new-line chars (as single entry); + - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` + could be used in regex to enable it; ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc From e8596cfce755080baee24ceda500b7deb81cb11a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 27 Mar 2017 11:27:41 +0200 Subject: [PATCH 27/76] amend resp. restore of change from 59c35bc44a175a672e084bc30511dfa3436ff052 (gh-129): - logging of "Log rotation detected" with new MSG level - introduces new log-level MSG (as INFO-2, 18) --- fail2ban/__init__.py | 2 ++ fail2ban/server/filter.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/__init__.py b/fail2ban/__init__.py index cd92dbab..7c752e24 100644 --- a/fail2ban/__init__.py +++ b/fail2ban/__init__.py @@ -34,7 +34,9 @@ Below derived from: https://mail.python.org/pipermail/tutor/2007-August/056243.html """ +logging.MSG = logging.INFO - 2 logging.NOTICE = logging.INFO + 5 +logging.addLevelName(logging.MSG, 'MSG') logging.addLevelName(logging.NOTICE, 'NOTICE') diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 459a47d0..066ee68f 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -820,7 +820,7 @@ class FileContainer: ## sys.stdout.flush() # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.info("Log rotation detected for %s" % self.__filename) + logSys.log(logging.MSG, "Log rotation detected for %s" % self.__filename) self.__hash = myHash self.__ino = stats.st_ino self.__pos = 0 From 7982d1e627913a8cde7bf840bbf3fcb8cb25deda Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 27 Mar 2017 11:31:41 +0200 Subject: [PATCH 28/76] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index e6114caa..213e2c0e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,7 @@ releases. - filter.d/domino-smtp: IBM Domino SMTP task (gh-1603) ### Enhancements +* Introduced new log-level `MSG` (as INFO-2, equivalent to 18) ver. 0.9.6 (2016/12/10) - stretch-is-coming From 7b93f111e1cd137a4d4935e276ae32b712d37008 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Tue, 28 Mar 2017 19:29:45 +0200 Subject: [PATCH 29/76] test_smtp inconsistency for py3+IPv6 It appears that, under Python3, on an IPv6 enabled machine, the testing SMTP server on 'localhost' can turn out to listen on ::1 only, which makes those tests break if the SMTP client part uses 127.0.0.1 directly. Using 'localhost' there as well makes the tests pass. --- fail2ban/tests/action_d/test_smtp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index d262f73e..d0858b85 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -67,7 +67,7 @@ class SMTPActionTest(unittest.TestCase): port = self.smtpd.socket.getsockname()[1] self.action = customActionModule.Action( - self.jail, "test", host="127.0.0.1:%i" % port) + self.jail, "test", host="localhost:%i" % port) ## because of bug in loop (see loop in asyncserver.py) use it's loop instead of asyncore.loop: self._active = True From 7437fbd75be2df301bdc9acfcf8e8d1381cbacda Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Mar 2017 18:54:28 +0200 Subject: [PATCH 30/76] strptime.py: small code review and performance optimization (get some properties on demand, etc.) --- fail2ban/server/strptime.py | 77 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index cdfe0e0e..d98b2e1b 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -95,18 +95,10 @@ def reGroupDictStrptime(found_dict, msec=False): Unix time stamp. """ - now = MyTime.now() - year = month = day = hour = minute = None - hour = minute = None + now = \ + year = month = day = hour = minute = tzoffset = \ + weekday = julian = week_of_year = None second = fraction = 0 - tzoffset = None - # Default to -1 to signify that values not known; not critical to have, - # though - week_of_year = -1 - week_of_year_start = -1 - # weekday and julian defaulted to -1 so as to signal need to calculate - # values - weekday = julian = -1 for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: @@ -199,31 +191,28 @@ def reGroupDictStrptime(found_dict, msec=False): # Fail2Ban will assume it's this year assume_year = False if year is None: + if not now: now = MyTime.now() year = now.year assume_year = True - # If we know the week of the year and what day of that week, we can figure - # out the Julian day of the year. - if julian == -1 and week_of_year != -1 and weekday != -1: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - # Cannot pre-calculate datetime.datetime() since can change in Julian - # calculation and thus could have different value for the day of the week - # calculation. - if julian != -1 and (month is None or day is None): - datetime_result = datetime.datetime.fromordinal((julian - 1) + datetime.datetime(year, 1, 1).toordinal()) - year = datetime_result.year - month = datetime_result.month - day = datetime_result.day - # Add timezone info - if tzoffset is not None: - gmtoff = tzoffset * 60 - else: - gmtoff = None + if month is None or day is None: + # If we know the week of the year and what day of that week, we can figure + # out the Julian day of the year. + if julian is None and week_of_year is not None and weekday is not None: + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + (week_of_year_start == 0)) + # Cannot pre-calculate datetime.datetime() since can change in Julian + # calculation and thus could have different value for the day of the week + # calculation. + if julian is not None: + datetime_result = datetime.datetime.fromordinal((julian - 1) + datetime.datetime(year, 1, 1).toordinal()) + year = datetime_result.year + month = datetime_result.month + day = datetime_result.day # Fail2Ban assume today assume_today = False if month is None and day is None: + if not now: now = MyTime.now() month = now.month day = now.day assume_today = True @@ -231,19 +220,25 @@ def reGroupDictStrptime(found_dict, msec=False): # Actully create date date_result = datetime.datetime( year, month, day, hour, minute, second, fraction) - if gmtoff is not None: - date_result = date_result - datetime.timedelta(seconds=gmtoff) + # Add timezone info + if tzoffset is not None: + date_result -= datetime.timedelta(seconds=tzoffset * 60) - if date_result > now and assume_today: - # Rollover at midnight, could mean it's yesterday... - date_result = date_result - datetime.timedelta(days=1) - if date_result > now and assume_year: - # Could be last year? - # also reset month and day as it's not yesterday... - date_result = date_result.replace( - year=year-1, month=month, day=day) + if assume_today: + if not now: now = MyTime.now() + if date_result > now: + # Rollover at midnight, could mean it's yesterday... + date_result -= datetime.timedelta(days=1) + if assume_year: + if not now: now = MyTime.now() + if date_result > now: + # Could be last year? + # also reset month and day as it's not yesterday... + date_result = date_result.replace( + year=year-1, month=month, day=day) - if gmtoff is not None: + # make time: + if tzoffset is not None: tm = calendar.timegm(date_result.utctimetuple()) else: tm = time.mktime(date_result.timetuple()) From ee3c9fcb75091b1d28cfcaf2535232ffe3dc465a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Mar 2017 22:07:40 +0200 Subject: [PATCH 31/76] "%y" - in the fail2ban parsed year without century should be always relative current century (>= 2000); cover several format specifiers and different "assume" cases (without year, without date, greater as now, etc.); --- fail2ban/server/strptime.py | 27 +++++------------ fail2ban/tests/datedetectortestcase.py | 41 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index d98b2e1b..55bdcc8c 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -108,13 +108,9 @@ def reGroupDictStrptime(found_dict, msec=False): # worthless without day of the week if key == 'y': year = int(val) - # Open Group specification for strptime() states that a %y - #value in the range of [00, 68] is in the century 2000, while - #[69,99] is in the century 1900 - if year <= 68: + # Fail2ban year should be always in the current century (>= 2000) + if year <= 2000: year += 2000 - else: - year += 1900 elif key == 'Y': year = int(val) elif key == 'm': @@ -148,7 +144,7 @@ def reGroupDictStrptime(found_dict, msec=False): elif key == 'S': second = int(val) elif key == 'f': - if msec: + if msec: # pragma: no cover - currently unused s = val # Pad to always return microseconds. s += "0" * (6 - len(s)) @@ -158,21 +154,14 @@ def reGroupDictStrptime(found_dict, msec=False): elif key == 'a': weekday = locale_time.a_weekday.index(val.lower()) elif key == 'w': - weekday = int(val) - if weekday == 0: - weekday = 6 - else: - weekday -= 1 + weekday = int(val) - 1 + if weekday < 0: weekday = 6 elif key == 'j': julian = int(val) elif key in ('U', 'W'): week_of_year = int(val) - if key == 'U': - # U starts week on Sunday. - week_of_year_start = 6 - else: - # W starts week on Monday. - week_of_year_start = 0 + # U starts week on Sunday, W - on Monday + week_of_year_start = 6 if key == 'U' else 0 elif key == 'z': z = val if z in ("Z", "UTC", "GMT"): @@ -242,6 +231,6 @@ def reGroupDictStrptime(found_dict, msec=False): tm = calendar.timegm(date_result.utctimetuple()) else: tm = time.mktime(date_result.timetuple()) - if msec: + if msec: # pragma: no cover - currently unused tm += fraction/1000000.0 return tm diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 5b32a7e9..39ab7173 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -298,6 +298,16 @@ iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + unittest.TestCase.setUp(self) + setUpMyTime() + + def tearDown(self): + """Call after every test case.""" + unittest.TestCase.tearDown(self) + tearDownMyTime() + def testIso8601(self): date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00Z")[0]) @@ -411,6 +421,37 @@ class CustomDateFormatsTest(unittest.TestCase): else: self.assertEqual(date, None) + def testVariousFormatSpecs(self): + for (matched, dp, line) in ( + # cover %B (full-month-name) and %I (as 12 == 0): + (1106438399.0, "^%B %Exd %I:%ExM:%ExS**", 'January 23 12:59:59'), + # cover %U (week of year starts on sunday) and %A (weekday): + (985208399.0, "^%y %U %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'), + # cover %W (week of year starts on monday) and %A (weekday): + (984603599.0, "^%y %W %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'), + # cover %W (week of year starts on monday) and %w (weekday, 0 - sunday): + (984949199.0, "^%y %W %w %ExH:%ExM:%ExS**", '01 11 0 21:59:59'), + # cover %W (week of year starts on monday) and %w (weekday, 6 - saturday): + (984862799.0, "^%y %W %w %ExH:%ExM:%ExS**", '01 11 6 21:59:59'), + # cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> back to yesterday (13 Aug): + (1123963199.0, "^%ExH:%ExM:%ExS**", '21:59:59'), + # cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> today (14 Aug): + (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), + # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): + (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), + ): + logSys.debug('== test: %r', (matched, dp, line)) + dd = DateDetector() + dd.appendTemplate(dp) + date = dd.getTime(line) + if matched: + self.assertTrue(date) + if isinstance(matched, basestring): # pragma: no cover + self.assertEqual(matched, date[1].group(1)) + else: + self.assertEqual(matched, date[0]) + else: # pragma: no cover + self.assertEqual(date, None) # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") From 05f5c6efccc22a728cd0d21659747f9335fa1daf Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 29 Mar 2017 12:32:34 +0200 Subject: [PATCH 32/76] Update README.md added wiki-reference; fixed mail-representation (after github swiched markdown syntax) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee654acb..72c48378 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ mechanisms if you really want to protect services. ------|------ This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs -are available in fail2ban(1) manpage and on the website http://www.fail2ban.org +are available in fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki) +and on the website http://www.fail2ban.org Installation: ------------- @@ -89,7 +90,7 @@ Contact: See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md) ### You just appreciate this program: -send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier )) +send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org)) or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) since Fail2Ban is "community-driven" for years now. From 8bf79fa4837a8c8f4d92e107fdf8585af41b434e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 17:44:15 +0200 Subject: [PATCH 33/76] implemented execution of `actionstart` on demand, if action depends on `family` (closes gh-1741); new action parameter "actionstart_on_demand" (bool) can be set to prevent/allow starting action on demand (default retrieved automatically, if some conditional parameter `param?family=...` presents in action properties); --- config/action.d/pf.conf | 5 ++- fail2ban/client/actionreader.py | 7 ++- fail2ban/server/action.py | 73 ++++++++++++++++++++++++++----- fail2ban/tests/servertestcase.py | 74 ++++++++++++++++++++++++-------- 4 files changed, 125 insertions(+), 34 deletions(-) diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf index b7476fa2..deb38c09 100644 --- a/config/action.d/pf.conf +++ b/config/action.d/pf.conf @@ -18,6 +18,9 @@ actionstart = echo "table <-> persist counters" | pfctl -f- echo "block proto from <-> to " | pfctl -f- +# Option: start_on_demand - to start action on demand +# Example: `action=pf[actionstart_on_demand=true]` +actionstart_on_demand = false # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -71,8 +74,6 @@ tablename = f2b # protocol = tcp - - # Option: actiontype # Notes.: defines additions to the blocking rule # Values: leave empty to block all attempts from the host diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 0fd55f41..b85f22a0 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,6 +38,7 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], + "actionstart_on_demand": ["string", None], "actionstop": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], @@ -73,8 +74,10 @@ class ActionReader(DefinitionInitConfigReader): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) # type-convert only after combined (otherwise boolean converting prevents substitution): - if opts.get('norestored'): - opts['norestored'] = self._convert_to_boolean(opts['norestored']) + for o in ('norestored', 'actionstart_on_demand'): + if opts.get(o): + opts[o] = self._convert_to_boolean(opts[o]) + # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 1c5eb0c9..8afeea6e 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -50,6 +50,8 @@ allowed_ipv6 = True # capture groups from filter for map to ticket data: FCUSTAG_CRE = re.compile(r''); # currently uppercase only +CONDITIONAL_FAM_RE = re.compile(r"^(\w+)\?(family)=") + # New line, space ADD_REPL_TAGS = { "br": "\n", @@ -290,6 +292,7 @@ class CommandAction(ActionBase): super(CommandAction, self).__init__(jail, name) self.__init = 1 self.__properties = None + self.__started = {} self.__substCache = {} self.clearAllParams() self._logSys.debug("Created %s" % self.__class__) @@ -342,7 +345,11 @@ class CommandAction(ActionBase): def _substCache(self): return self.__substCache - def _executeOperation(self, tag, operation): + def _getOperation(self, tag, family): + return self.replaceTag(tag, self._properties, + conditional=('family=' + family), cache=self.__substCache) + + def _executeOperation(self, tag, operation, family=[]): """Executes the operation commands (like "actionstart", "actionstop", etc). Replace the tags in the action command with actions properties @@ -352,14 +359,14 @@ class CommandAction(ActionBase): res = True try: # common (resp. ipv4): - startCmd = self.replaceTag(tag, self._properties, - conditional='family=inet4', cache=self.__substCache) - if startCmd: - res &= self.executeCmd(startCmd, self.timeout) + startCmd = None + if not family or 'inet4' in family: + startCmd = self._getOperation(tag, 'inet4') + if startCmd: + res &= self.executeCmd(startCmd, self.timeout) # start ipv6 actions if available: - if allowed_ipv6: - startCmd6 = self.replaceTag(tag, self._properties, - conditional='family=inet6', cache=self.__substCache) + if allowed_ipv6 and (not family or 'inet6' in family): + startCmd6 = self._getOperation(tag, 'inet6') if startCmd6 and startCmd6 != startCmd: res &= self.executeCmd(startCmd6, self.timeout) if not res: @@ -367,13 +374,34 @@ class CommandAction(ActionBase): except ValueError as e: raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e)) - def start(self): + COND_FAMILIES = {'inet4':1, 'inet6':1} + + @property + def _startOnDemand(self): + """Checks the action depends on family (conditional)""" + v = self._properties.get('actionstart_on_demand') + if v is None: + v = False + for n in self._properties: + if CONDITIONAL_FAM_RE.match(n): + v = True + break + self._properties['actionstart_on_demand'] = v + return v + + def start(self, family=[]): """Executes the "actionstart" command. Replace the tags in the action command with actions properties and executes the resulting command. """ - return self._executeOperation('', 'starting') + if not family: + # check the action depends on family (conditional): + if self._startOnDemand: + return True + elif self.__started.get(family): + return True + return self._executeOperation('', 'starting', family=family) def ban(self, aInfo): """Executes the "actionban" command. @@ -387,6 +415,20 @@ class CommandAction(ActionBase): Dictionary which includes information in relation to the ban. """ + # if we should start the action on demand (conditional by family): + if self._startOnDemand: + family = aInfo.get('family') + if not self.__started.get(family): + self.start(family) + self.__started[family] = 1 + # mark also another families as "started" (-1), if they are equal + # (on demand, but the same for ipv4 and ipv6): + cmd = self._getOperation('', family) + for f in CommandAction.COND_FAMILIES: + if f != family and not self.__started.get(f): + if cmd == self._getOperation('', f): + self.__started[f] = -1 + # ban: if not self._processCmd('', aInfo): raise RuntimeError("Error banning %(ip)s" % aInfo) @@ -411,7 +453,16 @@ class CommandAction(ActionBase): Replaces the tags in the action command with actions properties and executes the resulting command. """ - return self._executeOperation('', 'stopping') + family = [] + # cumulate started families, if started on demand (conditional): + if self._startOnDemand: + for f in CommandAction.COND_FAMILIES: + if self.__started.get(f) == 1: # only real started: + family.append(f) + self.__started[f] = 0 + # if no started (on demand) actions: + if not family: return True + return self._executeOperation('', 'stopping', family=family) def reload(self, **kwargs): """Executes the "actionreload" command. diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 51ff8880..603bb69f 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1185,10 +1185,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-multiport -- ('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`iptables -w -N f2b-j-w-iptables-mp`", "`iptables -w -A f2b-j-w-iptables-mp -j RETURN`", "`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", + ), + 'ip6-start': ( "`ip6tables -w -N f2b-j-w-iptables-mp`", "`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", @@ -1223,10 +1225,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-allports -- ('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`iptables -w -N f2b-j-w-iptables-ap`", "`iptables -w -A f2b-j-w-iptables-ap -j RETURN`", "`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`", + ), + 'ip6-start': ( "`ip6tables -w -N f2b-j-w-iptables-ap`", "`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`", "`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`", @@ -1261,9 +1265,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-ipset-proto6 -- ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), - 'start': ( + 'ip4-start': ( "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", + ), + 'ip6-start': ( "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), @@ -1293,9 +1299,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-ipset-proto6-allports -- ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="INPUT"]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), - 'start': ( + 'ip4-start': ( "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", + ), + 'ip6-start': ( "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`", "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), @@ -1325,10 +1333,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables -- ('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`iptables -w -N f2b-j-w-iptables`", "`iptables -w -A f2b-j-w-iptables -j RETURN`", "`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`", + ), + 'ip6-start': ( "`ip6tables -w -N f2b-j-w-iptables`", "`ip6tables -w -A f2b-j-w-iptables -j RETURN`", "`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`", @@ -1363,10 +1373,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-new -- ('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`iptables -w -N f2b-j-w-iptables-new`", "`iptables -w -A f2b-j-w-iptables-new -j RETURN`", "`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", + ), + 'ip6-start': ( "`ip6tables -w -N f2b-j-w-iptables-new`", "`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`", "`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", @@ -1401,8 +1413,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-xt_recent-echo -- ('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="INPUT"]', { 'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'), - 'start': ( + 'ip4-start': ( "`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`", + ), + 'ip6-start': ( "`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`", ), 'stop': ( @@ -1431,7 +1445,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # pf default -- multiport on default port (tag set in jail.conf, but not in this test case) - ('j-w-pf', 'pf[name=%(__name__)s]', { + ('j-w-pf', 'pf[name=%(__name__)s, actionstart_on_demand=false]', { 'ip4': (), 'ip6': (), 'start': ( '`echo "table persist counters" | pfctl -f-`', @@ -1468,13 +1482,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip6-ban': ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",), 'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",), }), - # pf allports -- - ('j-w-pf-ap', 'pf[actiontype=][name=%(__name__)s]', { + # pf allports -- test additionally "actionstart_on_demand" was set to true + ('j-w-pf-ap', 'pf[actiontype=, actionstart_on_demand=true][name=%(__name__)s]', { 'ip4': (), 'ip6': (), - 'start': ( + 'ip4-start': ( '`echo "table persist counters" | pfctl -f-`', '`echo "block proto tcp from to any" | pfctl -f-`', ), + 'ip6-start': (), # the same as ipv4 'stop': ( '`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -f-`', '`pfctl -t f2b-j-w-pf-ap -T flush`', @@ -1490,10 +1505,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # firewallcmd-multiport -- ('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', { 'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", "`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + ), + 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", @@ -1528,10 +1545,12 @@ class ServerConfigReaderTests(LogCaptureTestCase): # firewallcmd-allports -- ('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', { 'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'), - 'start': ( + 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`", "`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`", + ), + 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-ap`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`", @@ -1566,9 +1585,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): # firewallcmd-ipset -- ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), - 'start': ( + 'ip4-start': ( "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`", "`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + ), + 'ip6-start': ( "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), @@ -1614,6 +1635,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): jails = server._Server__jails + tickets = { + 'ip4': BanTicket('192.0.2.1'), + 'ip6': BanTicket('2001:DB8::'), + } for jail, act, tests in testJailsActions: # print(jail, jails[jail]) for a in jails[jail].actions: @@ -1627,25 +1652,36 @@ class ServerConfigReaderTests(LogCaptureTestCase): # test start : self.pruneLog('# === start ===') action.start() - self.assertLogged(*tests['start'], all=True) + if tests.get('start'): + self.assertLogged(*tests['start'], all=True) + else: + self.assertNotLogged(*tests['ip4-start']+tests['ip6-start'], all=True) + ainfo = { + 'ip4': _actions.Actions.ActionInfo(tickets['ip4'], jails[jail]), + 'ip6': _actions.Actions.ActionInfo(tickets['ip6'], jails[jail]), + } # test ban ip4 : self.pruneLog('# === ban-ipv4 ===') - action.ban({'ip': IPAddr('192.0.2.1')}) + action.ban(ainfo['ip4']) + if tests.get('ip4-start'): self.assertLogged(*tests['ip4-start'], all=True) + if tests.get('ip6-start'): self.assertNotLogged(*tests['ip6-start'], all=True) self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test unban ip4 : self.pruneLog('# === unban ipv4 ===') - action.unban({'ip': IPAddr('192.0.2.1')}) + action.unban(ainfo['ip4']) self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test ban ip6 : self.pruneLog('# === ban ipv6 ===') - action.ban({'ip': IPAddr('2001:DB8::')}) + action.ban(ainfo['ip6']) + if tests.get('ip6-start'): self.assertLogged(*tests['ip6-start'], all=True) + if tests.get('ip4-start'): self.assertNotLogged(*tests['ip4-start'], all=True) self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test unban ip6 : self.pruneLog('# === unban ipv6 ===') - action.unban({'ip': IPAddr('2001:DB8::')}) + action.unban(ainfo['ip6']) self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test stop : From ca18270beb046279d0bf6dfd011fcedad59e20e5 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 18:02:21 +0200 Subject: [PATCH 34/76] fix artificial test cases ('family' becomes mandatory in the action info, but dict was supplied in the test case) --- fail2ban/tests/servertestcase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 603bb69f..060d8e40 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1074,16 +1074,16 @@ class ServerConfigReaderTests(LogCaptureTestCase): action.start() # test ban ip4 : logSys.debug('# === ban-ipv4 ==='); self.pruneLog() - action.ban({'ip': IPAddr('192.0.2.1')}) + action.ban({'ip': IPAddr('192.0.2.1'), 'family': 'inet4'}) # test unban ip4 : logSys.debug('# === unban ipv4 ==='); self.pruneLog() - action.unban({'ip': IPAddr('192.0.2.1')}) + action.unban({'ip': IPAddr('192.0.2.1'), 'family': 'inet4'}) # test ban ip6 : logSys.debug('# === ban ipv6 ==='); self.pruneLog() - action.ban({'ip': IPAddr('2001:DB8::')}) + action.ban({'ip': IPAddr('2001:DB8::'), 'family': 'inet6'}) # test unban ip6 : logSys.debug('# === unban ipv6 ==='); self.pruneLog() - action.unban({'ip': IPAddr('2001:DB8::')}) + action.unban({'ip': IPAddr('2001:DB8::'), 'family': 'inet6'}) # test stop : logSys.debug('# === stop ==='); self.pruneLog() action.stop() From daa13eb5ddbcd40750a4a8ee1e608341dd6005f5 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 18:33:33 +0200 Subject: [PATCH 35/76] no cover for unreachable and abstract --- fail2ban/server/action.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 8afeea6e..9d518cfd 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -203,17 +203,17 @@ class ActionBase(object): self._name = name self._logSys = getLogger("fail2ban.%s" % self.__class__.__name__) - def start(self): + def start(self): # pragma: no cover - abstract """Executed when the jail/action is started. """ pass - def stop(self): + def stop(self): # pragma: no cover - abstract """Executed when the jail/action is stopped. """ pass - def ban(self, aInfo): + def ban(self, aInfo): # pragma: no cover - abstract """Executed when a ban occurs. Parameters @@ -224,7 +224,7 @@ class ActionBase(object): """ pass - def unban(self, aInfo): + def unban(self, aInfo): # pragma: no cover - abstract """Executed when a ban expires. Parameters @@ -399,7 +399,7 @@ class CommandAction(ActionBase): # check the action depends on family (conditional): if self._startOnDemand: return True - elif self.__started.get(family): + elif self.__started.get(family): # pragma: no cover - normally unreachable return True return self._executeOperation('', 'starting', family=family) From 44a26c615923f99526dc92e6efe67c1c6f8fe83e Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 29 Mar 2017 23:14:33 +0200 Subject: [PATCH 36/76] Update ChangeLog amend to gh-1742 --- ChangeLog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index bd49286e..da4854b6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,7 @@ TODO: implementing of options resp. other tasks from PR #1346 - fixed using new tag `` (without external command execution) * fail2ban-regex: fixed matched output by multi-line (buffered) parsing * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422) +* fixed ipv6-action errors on systems not supporting ipv6 und umgekehrt (gh-1741) ### New Features * New Actions: @@ -81,6 +82,12 @@ TODO: implementing of options resp. other tasks from PR #1346 the parsing of log-entries contain new-line chars (as single entry); - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` could be used in regex to enable it; +* implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742): + - new action parameter `actionstart_on_demand` (bool) can be set to prevent/allow starting action + on demand (default retrieved automatically, if some conditional parameter `param?family=...` + presents in action properties), see `action.d/pf.conf` for example; + - additionally `actionstop` will be executed only for families previously executing `actionstart` + (starting on demand only) ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc From a1e9cc552c1f8c2afaed42954b58dc4afc12b70f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 23:05:52 +0200 Subject: [PATCH 37/76] bulk unban: introduced new command `actionflush`: executed in order to flush all bans at once (e. g. by unban all, reload with removing action, stop, shutdown the system); the actions having `actionflush` do not execute `actionunban` for each single ticket --- fail2ban/client/actionreader.py | 1 + fail2ban/server/action.py | 21 +++++++++++++++++++++ fail2ban/server/actions.py | 18 +++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index b85f22a0..ace0b898 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -40,6 +40,7 @@ class ActionReader(DefinitionInitConfigReader): "actionstart": ["string", None], "actionstart_on_demand": ["string", None], "actionstop": ["string", None], + "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 9d518cfd..d00458ba 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -281,6 +281,8 @@ class CommandAction(ActionBase): self.actioncheck = '' ## Command executed in order to restore sane environment in error case. self.actionrepair = '' + ## Command executed in order to flush all bans at once (e. g. by stop/shutdown the system). + self.actionflush = '' ## Command executed in order to stop the system. self.actionstop = '' ## Command executed in case of reloading action. @@ -447,6 +449,25 @@ class CommandAction(ActionBase): if not self._processCmd('', aInfo): raise RuntimeError("Error unbanning %(ip)s" % aInfo) + def flush(self): + """Executes the "actionflush" command. + + Command executed in order to flush all bans at once (e. g. by stop/shutdown + the system), instead of unbunning of each single ticket. + + Replaces the tags in the action command with actions properties + and executes the resulting command. + """ + family = [] + # cumulate started families, if started on demand (conditional): + if self._startOnDemand: + for f in CommandAction.COND_FAMILIES: + if self.__started.get(f) == 1: # only real started: + family.append(f) + # if no started (on demand) actions: + if not family: return True + return self._executeOperation('', 'flushing', family=family) + def stop(self): """Executes the "actionstop" command. diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3a85e569..e652872e 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -447,25 +447,37 @@ class Actions(JailThread, Mapping): If actions specified, don't flush list - just execute unban for given actions (reload, obsolete resp. removed actions). """ + log = True if actions is None: logSys.debug("Flush ban list") lst = self.__banManager.flushBanList() else: + log = False lst = iter(self.__banManager) cnt = 0 + # first we'll execute flush for actions supporting this operation: + unbactions = {} + for name, action in (actions if actions is not None else self._actions).iteritems(): + if hasattr(action, 'flush') and action.actionflush: + logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) + action.flush() + else: + unbactions[name] = action + actions = unbactions + # unban each ticket with non-flasheable actions: for ticket in lst: # delete ip from database also: if db and self._jail.database is not None: ip = str(ticket.getIP()) self._jail.database.delBan(self._jail, ip) # unban ip: - self.__unBan(ticket, actions=actions) + self.__unBan(ticket, actions=actions, log=log) cnt += 1 logSys.debug("Unbanned %s, %s ticket(s) in %r", cnt, self.__banManager.size(), self._jail.name) return cnt - def __unBan(self, ticket, actions=None): + def __unBan(self, ticket, actions=None, log=True): """Unbans host corresponding to the ticket. Executes the actions in order to unban the host given in the @@ -482,7 +494,7 @@ class Actions(JailThread, Mapping): unbactions = actions ip = ticket.getIP() aInfo = self.__getActionInfo(ticket) - if actions is None: + if log: logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) for name, action in unbactions.iteritems(): try: From d03872fbbf1b78d6047f4bc1876e2acc3293873d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 23:20:43 +0200 Subject: [PATCH 38/76] bulk unban: add new command `actionflush` default for several iptables/iptables-ipset actions (and common include): iptables-common iptables iptables-allports iptables-multiport-log iptables-multiport iptables-new iptables-ipset-proto4 iptables-ipset-proto6 iptables-ipset-proto6-allports executing `actionflush` command covered for this actions now --- config/action.d/iptables-allports.conf | 2 +- config/action.d/iptables-common.conf | 8 +++++ config/action.d/iptables-ipset-proto4.conf | 9 +++++- .../iptables-ipset-proto6-allports.conf | 8 ++++- config/action.d/iptables-ipset-proto6.conf | 8 ++++- config/action.d/iptables-multiport-log.conf | 10 +++++-- config/action.d/iptables-multiport.conf | 2 +- config/action.d/iptables-new.conf | 2 +- config/action.d/iptables-xt_recent-echo.conf | 6 ++++ config/action.d/iptables.conf | 2 +- fail2ban/tests/servertestcase.py | 29 +++++++++++++++++++ 11 files changed, 77 insertions(+), 9 deletions(-) diff --git a/config/action.d/iptables-allports.conf b/config/action.d/iptables-allports.conf index 15f3cbcc..dbea5984 100644 --- a/config/action.d/iptables-allports.conf +++ b/config/action.d/iptables-allports.conf @@ -26,7 +26,7 @@ actionstart = -N f2b- # Values: CMD # actionstop = -D -p -j f2b- - -F f2b- + -X f2b- # Option: actioncheck diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf index a3921021..e016ef2f 100644 --- a/config/action.d/iptables-common.conf +++ b/config/action.d/iptables-common.conf @@ -16,6 +16,14 @@ after = iptables-blocktype.local iptables-common.local # iptables-blocktype.local is obsolete +[Definition] + +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = -F f2b- + [Init] diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index 2f63cd4b..30353f36 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -30,12 +30,19 @@ before = iptables-common.conf actionstart = ipset --create f2b- iphash -I -p -m multiport --dports -m set --match-set f2b- src -j + +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = ipset --flush f2b- + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = -D -p -m multiport --dports -m set --match-set f2b- src -j - ipset --flush f2b- + ipset --destroy f2b- # Option: actionban diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index 113f599e..b761ad8c 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -29,12 +29,18 @@ before = iptables-common.conf actionstart = ipset create hash:ip timeout -I -m set --match-set src -j +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = ipset flush + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = -D -m set --match-set src -j - ipset flush + ipset destroy # Option: actionban diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index dee7b029..e337eedf 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -29,12 +29,18 @@ before = iptables-common.conf actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = ipset flush + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = -D -p -m multiport --dports -m set --match-set src -j - ipset flush + ipset destroy # Option: actionban diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf index 1777ce62..62c2b4b1 100644 --- a/config/action.d/iptables-multiport-log.conf +++ b/config/action.d/iptables-multiport-log.conf @@ -26,13 +26,19 @@ actionstart = -N f2b- -I f2b--log -j LOG --log-prefix "$(expr f2b- : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2 -A f2b--log -j +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = -F f2b- + -F f2b--log + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = -D -p -m multiport --dports -j f2b- - -F f2b- - -F f2b--log + -X f2b- -X f2b--log diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf index 9fd87d20..c05f6ffc 100644 --- a/config/action.d/iptables-multiport.conf +++ b/config/action.d/iptables-multiport.conf @@ -23,7 +23,7 @@ actionstart = -N f2b- # Values: CMD # actionstop = -D -p -m multiport --dports -j f2b- - -F f2b- + -X f2b- # Option: actioncheck diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf index 795bc601..5b316807 100644 --- a/config/action.d/iptables-new.conf +++ b/config/action.d/iptables-new.conf @@ -25,7 +25,7 @@ actionstart = -N f2b- # Values: CMD # actionstop = -D -m state --state NEW -p --dport -j f2b- - -F f2b- + -X f2b- # Option: actioncheck diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf index 018d2cf6..1970de14 100644 --- a/config/action.d/iptables-xt_recent-echo.conf +++ b/config/action.d/iptables-xt_recent-echo.conf @@ -35,6 +35,12 @@ before = iptables-common.conf # shorter of the two timeouts actually matters. actionstart = if [ `id -u` -eq 0 ];then -I -m recent --update --seconds 3600 --name -j ;fi +# Option: actionflush +# +# [TODO] Flushing is currently not implemented for xt_recent +# +actionflush = + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf index 38985ffa..bf83e24a 100644 --- a/config/action.d/iptables.conf +++ b/config/action.d/iptables.conf @@ -23,7 +23,7 @@ actionstart = -N f2b- # Values: CMD # actionstop = -D -p --dport -j f2b- - -F f2b- + -X f2b- # Option: actioncheck diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 060d8e40..604a15ee 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1195,6 +1195,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", ), + 'flush': ( + "`iptables -w -F f2b-j-w-iptables-mp`", + "`ip6tables -w -F f2b-j-w-iptables-mp`", + ), 'stop': ( "`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", "`iptables -w -F f2b-j-w-iptables-mp`", @@ -1235,6 +1239,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`", "`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`", ), + 'flush': ( + "`iptables -w -F f2b-j-w-iptables-ap`", + "`ip6tables -w -F f2b-j-w-iptables-ap`", + ), 'stop': ( "`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`", "`iptables -w -F f2b-j-w-iptables-ap`", @@ -1273,6 +1281,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), + 'flush': ( + "`ipset flush f2b-j-w-iptables-ipset`", + "`ipset flush f2b-j-w-iptables-ipset6`", + ), 'stop': ( "`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset`", @@ -1307,6 +1319,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`", "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), + 'flush': ( + "`ipset flush f2b-j-w-iptables-ipset-ap`", + "`ipset flush f2b-j-w-iptables-ipset-ap6`", + ), 'stop': ( "`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset-ap`", @@ -1343,6 +1359,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ip6tables -w -A f2b-j-w-iptables -j RETURN`", "`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`", ), + 'flush': ( + "`iptables -w -F f2b-j-w-iptables`", + "`ip6tables -w -F f2b-j-w-iptables`", + ), 'stop': ( "`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`", "`iptables -w -F f2b-j-w-iptables`", @@ -1383,6 +1403,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`", "`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", ), + 'flush': ( + "`iptables -w -F f2b-j-w-iptables-new`", + "`ip6tables -w -F f2b-j-w-iptables-new`", + ), 'stop': ( "`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", "`iptables -w -F f2b-j-w-iptables-new`", @@ -1684,6 +1708,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): action.unban(ainfo['ip6']) self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) + # test flush for actions should supported this: + if tests.get('flush'): + self.pruneLog('# === flush ===') + action.flush() + self.assertLogged(*tests['flush'], all=True) # test stop : self.pruneLog('# === stop ===') action.stop() From 042a060a54ed517e91496073fb8f7ad66ee7a78e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Mar 2017 23:23:32 +0200 Subject: [PATCH 39/76] additionally complex test-case coverage for `actionflush` inside server via actions-mechanism of fail2ban - reload with removing action, unban all, stopping of jails and actions, etc. --- fail2ban/tests/fail2banclienttestcase.py | 41 ++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 35e00421..3e046042 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -762,6 +762,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "norestored = %(_exec_once)s", "restore = ", "info = ", + "_use_flush_ = echo [] : -- flushing IPs", "actionstart = echo '[%(name)s] %(actname)s: ** start'", start, "actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload, "actionban = echo '[%(name)s] %(actname)s: ++ ban %(restore)s%(info)s'", ban, @@ -788,7 +789,8 @@ class Fail2banServerTest(Fail2banClientServerBase): if 1 in actions else "", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" \ + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: '," + " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test1log, " " + test2log if 2 in enabled else "", @@ -802,7 +804,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" \ + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" + " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, "enabled = true" if 2 in enabled else "", @@ -874,6 +877,12 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Creating new jail 'test-jail2'", "Jail 'test-jail2' started", all=True) + # test action3 removed, test flushing successful (and no single unban occurred): + self.assertLogged( + "stdout: '[test-jail1] test-action3: -- flushing IPs'", + "stdout: '[test-jail1] test-action3: __ stop'", all=True) + self.assertNotLogged( + "stdout: '[test-jail1] test-action3: -- unban 192.0.2.1'") # update action1, delete action2 (should be stopped via configuration)... self.pruneLog("[test-phase 2a]") @@ -969,12 +978,18 @@ class Fail2banServerTest(Fail2banClientServerBase): "stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'", all=True) - # don't need actions anymore: - _write_action_cfg(actname="test-action2", allow=False) - _write_jail_cfg(actions=[]) + # ban manually to test later flush by unban all: + self.pruneLog("[test-phase 2d]") + self.execSuccess(startparams, + "set", "test-jail2", "banip", "192.0.2.21") + self.execSuccess(startparams, + "set", "test-jail2", "banip", "192.0.2.22") + self.assertLogged( + "stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22", + "stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22 ", all=True, wait=MID_WAITTIME) # restart jail with unban all: - self.pruneLog("[test-phase 2d]") + self.pruneLog("[test-phase 2e]") self.execSuccess(startparams, "restart", "--unban", "test-jail2") self.assertLogged( @@ -986,12 +1001,26 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail2] Unban 192.0.2.4", "[test-jail2] Unban 192.0.2.8", all=True ) + # test unban (action2): + self.assertLogged( + "stdout: '[test-jail2] test-action2: -- unban 192.0.2.21", + "stdout: '[test-jail2] test-action2: -- unban 192.0.2.22'", all=True) + # test flush (action3, and no single unban via action3 occurred): + self.assertLogged( + "stdout: '[test-jail2] test-action3: -- flushing IPs'") + self.assertNotLogged( + "stdout: '[test-jail2] test-action3: -- unban 192.0.2.21'", + "stdout: '[test-jail2] test-action3: -- unban 192.0.2.22'", all=True) # no more ban (unbanned all): self.assertNotLogged( "[test-jail2] Ban 192.0.2.4", "[test-jail2] Ban 192.0.2.8", all=True ) + # don't need actions anymore: + _write_action_cfg(actname="test-action2", allow=False) + _write_jail_cfg(actions=[]) + # reload jail1 without restart (without ban/unban): self.pruneLog("[test-phase 3]") self.execSuccess(startparams, "reload", "test-jail1") From 97e8b42d342c437d27d43fa4ed136c11a5fa1be4 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 30 Mar 2017 13:02:37 +0200 Subject: [PATCH 40/76] dummy action extended with more examples and test-covered now --- config/action.d/dummy.conf | 26 +++++++++++++++++++++----- fail2ban/server/actions.py | 2 +- fail2ban/tests/servertestcase.py | 27 +++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/config/action.d/dummy.conf b/config/action.d/dummy.conf index dc4e1dbf..41250c27 100644 --- a/config/action.d/dummy.conf +++ b/config/action.d/dummy.conf @@ -10,14 +10,23 @@ # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = touch /var/run/fail2ban/fail2ban.dummy - printf %%b "\n" >> /var/run/fail2ban/fail2ban.dummy +actionstart = if [ ! -z '' ]; then touch ; fi; + printf %%b "\n" + echo "%(debug)s started" + +# Option: actionflush +# Notes.: command executed once to flush (clear) all IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = printf %%b "-*\n" + echo "%(debug)s clear all" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = rm -f /var/run/fail2ban/fail2ban.dummy +actionstop = if [ ! -z '' ]; then rm -f ; fi; + echo "%(debug)s stopped" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -31,7 +40,8 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "+\n" >> /var/run/fail2ban/fail2ban.dummy +actionban = printf %%b "+\n" + echo "%(debug)s banned (family: )" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -39,9 +49,15 @@ actionban = printf %%b "+\n" >> /var/run/fail2ban/fail2ban.dummy # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = printf %%b "-\n" >> /var/run/fail2ban/fail2ban.dummy +actionunban = printf %%b "-\n" + echo "%(debug)s unbanned (family: )" + + +debug = [] -- [Init] init = 123 +target = /var/run/fail2ban/fail2ban.dummy +to_target = >> diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index e652872e..c33359c9 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -452,7 +452,7 @@ class Actions(JailThread, Mapping): logSys.debug("Flush ban list") lst = self.__banManager.flushBanList() else: - log = False + log = False # don't log "[jail] Unban ..." if removing actions only. lst = iter(self.__banManager) cnt = 0 # first we'll execute flush for actions supporting this operation: diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 604a15ee..1644d895 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1182,6 +1182,33 @@ class ServerConfigReaderTests(LogCaptureTestCase): # 'start', 'stop' - should be found (logged) on action start/stop, # etc. testJailsActions = ( + # dummy -- + ('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', { + 'ip4': ('family: inet4',), 'ip6': ('family: inet6',), + 'start': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`', + ), + 'flush': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- clear all"`', + ), + 'stop': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- stopped"`', + ), + 'ip4-check': (), + 'ip6-check': (), + 'ip4-ban': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 192.0.2.1 (family: inet4)"`', + ), + 'ip4-unban': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 192.0.2.1 (family: inet4)"`', + ), + 'ip6-ban': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 2001:db8:: (family: inet6)"`', + ), + 'ip6-unban': ( + '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 2001:db8:: (family: inet6)"`', + ), + }), # iptables-multiport -- ('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), From e7f1fc5cb360a2aebcb71c037ec32b0a889ab3f1 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 31 Mar 2017 10:39:50 +0200 Subject: [PATCH 41/76] Update ChangeLog enhancements of #1743 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index da4854b6..d644f330 100644 --- a/ChangeLog +++ b/ChangeLog @@ -88,6 +88,10 @@ TODO: implementing of options resp. other tasks from PR #1346 presents in action properties), see `action.d/pf.conf` for example; - additionally `actionstop` will be executed only for families previously executing `actionstart` (starting on demand only) +* introduced new command `actionflush`: executed in order to flush all bans at once + e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743), + the actions having `actionflush` do not execute `actionunban` for each single ticket +* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include); ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc From 4fc6323ff038439e8b7d70ecf7d09488e0ac79a9 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Fri, 7 Apr 2017 13:59:22 +0200 Subject: [PATCH 42/76] haproxy-http-auth: avoid port number in IPv6 addresses The solution taken is to consume the port number explicitely in the regexp. --- ChangeLog | 1 + config/filter.d/haproxy-http-auth.conf | 2 +- fail2ban/tests/files/logs/haproxy-http-auth | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d644f330..07a144f4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,7 @@ TODO: implementing of options resp. other tasks from PR #1346 * filter.d/sendmail-reject.conf: - rewritten using `prefregex` and used MLFID-related multi-line parsing; - optional parameter `mode` introduced: normal (default), extra or aggressive +* filter.d/haproxy-http-auth: do not mistake client port for part of an IPv6 address (gh-1745) * `action.d/complain.conf` - fixed using new tag `` (sh/dash compliant now) * `action.d/sendmail-geoip-lines.conf` diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf index 298ca292..d85f5e9e 100644 --- a/config/filter.d/haproxy-http-auth.conf +++ b/config/filter.d/haproxy-http-auth.conf @@ -28,7 +28,7 @@ _daemon = haproxy # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)s.* -1/-1/-1/-1/\+*\d* 401 +failregex = ^%(__prefix_line)s:\d+.* -1/-1/-1/-1/\+*\d* 401 # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth index 298f1972..5a5141fc 100644 --- a/fail2ban/tests/files/logs/haproxy-http-auth +++ b/fail2ban/tests/files/logs/haproxy-http-auth @@ -2,3 +2,5 @@ Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439] main app/app1 1939/0/1/0/1940 403 5168 - - ---- 3/3/0/0/0 0/0 "GET / HTTP/1.1" # failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" } Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" +# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "2001:db8::1234" } +Nov 14 22:45:11 test haproxy[760]: 2001:db8::1234:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" From 07023436aca2a71ba338fe9f342c0908bd0b6e39 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Fri, 7 Apr 2017 14:04:09 +0200 Subject: [PATCH 43/76] haproxy-http-auth: added a test for IPv4-mapped-in-IPv6 This what one gets in logis if haproxy is binding to :: on a dual-stack system. --- fail2ban/tests/files/logs/haproxy-http-auth | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth index 5a5141fc..403a8083 100644 --- a/fail2ban/tests/files/logs/haproxy-http-auth +++ b/fail2ban/tests/files/logs/haproxy-http-auth @@ -4,3 +4,5 @@ Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439] Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" # failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "2001:db8::1234" } Nov 14 22:45:11 test haproxy[760]: 2001:db8::1234:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" +# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" } +Nov 14 22:45:11 test haproxy[760]: ::ffff:192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" From 4f0f22702ac795b53d8c953992e330c85e749142 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 11 Apr 2017 09:11:08 +0200 Subject: [PATCH 44/76] Update haproxy-http-auth.conf little bit more precise expression --- config/filter.d/haproxy-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf index d85f5e9e..f92f9d67 100644 --- a/config/filter.d/haproxy-http-auth.conf +++ b/config/filter.d/haproxy-http-auth.conf @@ -28,7 +28,7 @@ _daemon = haproxy # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)s:\d+.* -1/-1/-1/-1/\+*\d* 401 +failregex = ^%(__prefix_line)s(?::\d+)?\s+.* -1/-1/-1/-1/\+*\d* 401 # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From bb79e7f41327983df068119ec48b90a5d0ceab47 Mon Sep 17 00:00:00 2001 From: Peter van der Does Date: Tue, 11 Apr 2017 11:13:58 -0400 Subject: [PATCH 45/76] Parameter not needed The parameter '-s' causes an error as the already has the parameter. --- config/action.d/mail-whois-lines.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 0852ba8f..7ebb8b9f 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -21,7 +21,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban" | -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban" | "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -30,7 +30,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban" | -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban" | "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command From a639f0b083c213bde4ff3dcfbbb9fbcab0dd55f8 Mon Sep 17 00:00:00 2001 From: Paul Brook Date: Sun, 16 Apr 2017 12:11:05 -0400 Subject: [PATCH 46/76] BF: specify explicit time offset not a time zone name to avoid needing tzdata during testing --- fail2ban/tests/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 9155dc37..54861050 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -95,7 +95,10 @@ def setUpMyTime(): # Set the time to a fixed, known value # Sun Aug 14 12:00:00 CEST 2005 # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match - os.environ['TZ'] = 'Europe/Zurich' + # This offset corresponds to Europe/Zurich timezone. Specifying it + # explicitly allows to avoid requiring tzdata package to be installed during + # testing. See https://bugs.debian.org/855920 for more information + os.environ['TZ'] = 'CET-01CEST-02,M3.5.0,M10.5.0' time.tzset() MyTime.setTime(1124013600) From 17922b621c2d8dc99120470647ece16c56f298d8 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 20 Apr 2017 15:23:59 +0200 Subject: [PATCH 47/76] Update ChangeLog replaced german in entry ;) --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 07a144f4..17670805 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,7 +30,7 @@ TODO: implementing of options resp. other tasks from PR #1346 - fixed using new tag `` (without external command execution) * fail2ban-regex: fixed matched output by multi-line (buffered) parsing * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422) -* fixed ipv6-action errors on systems not supporting ipv6 und umgekehrt (gh-1741) +* fixed ipv6-action errors on systems not supporting ipv6 and vice versa (gh-1741) ### New Features * New Actions: From 6dfd080e20d1e088b5d840933aa864b88fb77d24 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 21 Apr 2017 11:17:13 +0200 Subject: [PATCH 48/76] Update apache-auth.conf remove forgotten referer, that may prevent failure recognition (belongs to gh-1645) --- config/filter.d/apache-auth.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index f301e290..e22a49b7 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -21,8 +21,8 @@ failregex = ^client denied by server configuration\b ^Authorization of user (?:\S*|.*?) to access .*? failed\b ^%(auth_type)suser (?:\S*|.*?): password mismatch\b ^%(auth_type)suser `(?:[^']*|.*?)' in realm `.+' (not found|denied by provider)\b - ^%(auth_type)sinvalid nonce .* received - length is not \S+(, referer: \S+)?\s*$ - ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected `.+'(, referer: \S+)?\s*$ + ^%(auth_type)sinvalid nonce .* received - length is not\b + ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected\b ^%(auth_type)sunknown algorithm `(?:[^']*|.*?)' received\b ^invalid qop `(?:[^']*|.*?)' received\b ^%(auth_type)sinvalid nonce .*? received - user attempted time travel\b From e35ed1cdf7222a1dc1c8caaba08568b78d703a04 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 21 Apr 2017 11:24:32 +0200 Subject: [PATCH 49/76] Update ChangeLog Changes of #1645 --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 17670805..9e722608 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ TODO: implementing of options resp. other tasks from PR #1346 documentation should be extended (new options, etc) ### Fixes +* `filter.d/apache-auth.conf`: + - better failure recognition using short form of regex (url/referer are foreign inputs, see gh-1645) * `filter.d/pam-generic.conf`: - [grave] injection on user name to host fixed * `filter.d/sshd.conf`: From 507034c5be77c7894e2eee8afc2ce4ea84002674 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Apr 2017 15:32:44 +0200 Subject: [PATCH 50/76] filter.d/apache-auth.conf: joined some similar expressions --- config/filter.d/apache-auth.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index e22a49b7..d9a6fa5e 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -14,10 +14,8 @@ prefregex = ^%(_apache_error_client)s (?:AH\d+: )?.+$ # auth_type = ((?:Digest|Basic): )? auth_type = ([A-Z]\w+: )? -failregex = ^client denied by server configuration\b - ^user (?:\S*|.*?) auth(?:oriz|entic)ation failure\b - ^user (?:\S*|.*?) not found\b - ^client used wrong authentication scheme\b +failregex = ^client (?:denied by server configuration|used wrong authentication scheme)\b + ^user (?:\S*|.*?) (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b ^Authorization of user (?:\S*|.*?) to access .*? failed\b ^%(auth_type)suser (?:\S*|.*?): password mismatch\b ^%(auth_type)suser `(?:[^']*|.*?)' in realm `.+' (not found|denied by provider)\b From 3161bcf78b86ee5916d85c9551ba2efedede9d84 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Apr 2017 19:13:38 +0200 Subject: [PATCH 51/76] filter.d/exim.conf: optional part `(...)` after host-name before `[IP]`, normalized over whole config file. # Conflicts: # config/filter.d/exim.conf --- config/filter.d/exim.conf | 6 +++--- fail2ban/tests/files/logs/exim | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index a1d699c0..27d73426 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -14,13 +14,13 @@ before = exim-common.conf [Definition] failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ - ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ - ^%(pid)s no MAIL in SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$ - ^%(pid)s \S+ SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ + ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$ + ^%(pid)s ([\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 9053bf8d..44d9e34b 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -67,3 +67,6 @@ 2016-04-01 11:09:21 [18648] SMTP protocol error in "AUTH LOGIN" H=host.example.com (SERVER) [192.0.2.1]:4692 I=[172.89.0.6]:25 AUTH command used when not advertised # failJSON: { "time": "2016-03-27T16:48:48", "match": true , "host": "192.0.2.1" } 2016-03-27 16:48:48 [21478] 1akDqs-0005aQ-9b SMTP connection from host.example.com (SERVER) [192.0.2.1]:47714 I=[172.89.0.6]:25 closed by DROP in ACL + +# failJSON: { "time": "2017-04-23T22:45:59", "match": true , "host": "192.0.2.2", "desc": "optional part (...)" } +2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com) From 99344d28c8fe258de7fb72810ad5646ab62dbbc7 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Apr 2017 20:17:57 +0200 Subject: [PATCH 52/76] Introduces new tags with hostname: - `` - fully-qualified name of host (the same as `$(hostname -f)`) - `` - short hostname (the same as `$(uname -n)`) Execution of `uname -n` replaced in all mail actions with most interesting fully-qualified ``. --- config/action.d/mail-buffered.conf | 6 +++--- config/action.d/mail-whois-lines.conf | 6 +++--- config/action.d/mail-whois.conf | 6 +++--- config/action.d/mail.conf | 6 +++--- config/action.d/sendmail-buffered.conf | 8 +++---- config/action.d/sendmail-common.conf | 4 ++-- config/action.d/sendmail-geoip-lines.conf | 2 +- .../sendmail-whois-ipjailmatches.conf | 2 +- config/action.d/sendmail-whois-ipmatches.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-matches.conf | 2 +- config/action.d/sendmail-whois.conf | 2 +- config/action.d/sendmail.conf | 2 +- config/action.d/xarf-login-attack.conf | 4 ++-- config/jail.conf | 2 +- fail2ban/server/actions.py | 4 ++++ fail2ban/server/ipdns.py | 21 +++++++++++++++++++ 17 files changed, 53 insertions(+), 28 deletions(-) diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf index e74db9cc..88cd623f 100644 --- a/config/action.d/mail-buffered.conf +++ b/config/action.d/mail-buffered.conf @@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Output will be buffered until lines are available.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -28,13 +28,13 @@ actionstop = if [ -f ]; then These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary from `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : Summary from " rm fi printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 7ebb8b9f..37e2d9b0 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -21,7 +21,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban" | "[Fail2Ban] : started on `uname -n`" + Fail2Ban" | "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -30,7 +30,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban" | "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban" | "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -56,7 +56,7 @@ _ban_mail_content = ( printf %%b "Hi,\n Regards,\n Fail2Ban" ) -actionban = %(_ban_mail_content)s | "[Fail2Ban] : banned from `uname -n`" +actionban = %(_ban_mail_content)s | "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index 553bfb69..1f69f4c6 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -20,7 +20,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n Here is more information about :\n `%(_whois_command)s`\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf index 4715ecc5..cfc1cf65 100644 --- a/config/action.d/mail.conf +++ b/config/action.d/mail.conf @@ -16,7 +16,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban"|mail -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf index a91a6957..37bc642d 100644 --- a/config/action.d/sendmail-buffered.conf +++ b/config/action.d/sendmail-buffered.conf @@ -17,7 +17,7 @@ norestored = 1 # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` +actionstart = printf %%b "Subject: [Fail2Ban] : started on From: <> To: \n Hi,\n @@ -31,7 +31,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` # Values: CMD # actionstop = if [ -f ]; then - printf %%b "Subject: [Fail2Ban] : summary from `uname -n` + printf %%b "Subject: [Fail2Ban] : summary from From: <> To: \n Hi,\n @@ -41,7 +41,7 @@ actionstop = if [ -f ]; then Fail2Ban" | /usr/sbin/sendmail -f rm fi - printf %%b "Subject: [Fail2Ban] : stopped on `uname -n` + printf %%b "Subject: [Fail2Ban] : stopped on From: Fail2Ban <> To: \n Hi,\n @@ -64,7 +64,7 @@ actioncheck = actionban = printf %%b "`date`: ( failures)\n" >> LINE=$( wc -l | awk '{ print $1 }' ) if [ $LINE -ge ]; then - printf %%b "Subject: [Fail2Ban] : summary from `uname -n` + printf %%b "Subject: [Fail2Ban] : summary from From: <> To: \n Hi,\n diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf index 1475dedb..46eca9ca 100644 --- a/config/action.d/sendmail-common.conf +++ b/config/action.d/sendmail-common.conf @@ -14,7 +14,7 @@ after = sendmail-common.local # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` +actionstart = printf %%b "Subject: [Fail2Ban] : started on Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n @@ -27,7 +27,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on `uname -n` # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = printf %%b "Subject: [Fail2Ban] : stopped on `uname -n` +actionstop = printf %%b "Subject: [Fail2Ban] : stopped on Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index decf2c05..b7c1bf36 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -23,7 +23,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 5bcefe89..06ea3a3e 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -19,7 +19,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index 4a8edcb7..83bff1b4 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -19,7 +19,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index e3a1c974..4b947cb2 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -20,7 +20,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index fc4ba061..01520135 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -19,7 +19,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index b8d99423..2fb01ed3 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -19,7 +19,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index 62c94439..cf420915 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -19,7 +19,7 @@ norestored = 1 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = printf %%b "Subject: [Fail2Ban] : banned from Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 5274cdaf..9d441aa3 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -46,7 +46,7 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(di FROM= SERVICE= FAILURES= - REPORTID=