From e4a215ca5005ed0fe0b4477d2ec1b1226dcb06c2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 31 Dec 2013 19:00:26 +1100 Subject: [PATCH 1/6] BF: fix infinite recursion case in Action.substituteRecursiveTags --- fail2ban/server/action.py | 14 +++++++++++--- fail2ban/tests/actiontestcase.py | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 17f33a6f..7d107d9f 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -276,19 +276,27 @@ class Action: for tag, value in tags.iteritems(): value = str(value) m = t.search(value) + done = [] + #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: - if m.group(1) == tag: + found_tag = m.group(1) + #logSys.log(5, 'found: %s' % found_tag) + if found_tag == tag or found_tag in done: # recursive definitions are bad + #logSys.log(5, 'recursion fail') return False else: - if tags.has_key(m.group(1)): - value = value[0:m.start()] + tags[m.group(1)] + value[m.end():] + if tags.has_key(found_tag): + value = value[0:m.start()] + tags[found_tag] + value[m.end():] + #logSys.log(5, 'value now: %s' % value) + done.append(found_tag) m = t.search(value, m.start()) else: # Missing tags are ok so we just continue on searching. # cInfo can contain aInfo elements like and valid shell # constructs like . m = t.search(value, m.start() + 1) + #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) tags[tag] = value return tags substituteRecursiveTags = staticmethod(substituteRecursiveTags) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 36ecc3c9..c9364c07 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -58,6 +58,9 @@ class ExecuteAction(LogCaptureTestCase): self.assertFalse(Action.substituteRecursiveTags({'A': ''})) self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': ''})) self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) + # part recursion + self.assertFalse(Action.substituteRecursiveTags({'A': 'to= fromip=', 'C': '', 'B': '', 'D': ''})) + self.assertFalse(Action.substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) # missing tags are ok self.assertEqual(Action.substituteRecursiveTags({'A': ''}), {'A': ''}) self.assertEqual(Action.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) From a4c38439df0b882e6b7333b00f3690966862557f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 31 Dec 2013 19:01:21 +1100 Subject: [PATCH 2/6] ENH: add substition tags to filter definitions. Closes gh-539 --- MANIFEST | 1 + fail2ban/client/filterreader.py | 11 ++++++-- fail2ban/tests/clientreadertestcase.py | 28 +++++++++++++++++++ fail2ban/tests/files/filter.d/substition.conf | 8 ++++++ 4 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 fail2ban/tests/files/filter.d/substition.conf diff --git a/MANIFEST b/MANIFEST index 0b339b89..b03d309f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -80,6 +80,7 @@ fail2ban/tests/files/config/apache-auth/README fail2ban/tests/files/config/apache-auth/noentry/.htaccess fail2ban/tests/files/database_v1.db fail2ban/tests/files/ignorecommand.py +fail2ban/tests/files/filter.d/substition.conf fail2ban/tests/files/filter.d/testcase-common.conf fail2ban/tests/files/filter.d/testcase01.conf fail2ban/tests/files/testcase01.log diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index d8a6dbe8..8ea854b6 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -26,6 +26,7 @@ __license__ = "GPL" import logging, os, shlex from configreader import ConfigReader, DefinitionInitConfigReader +from fail2ban.server.action import Action # Gets the instance of the logger. logSys = logging.getLogger(__name__) @@ -42,14 +43,18 @@ class FilterReader(DefinitionInitConfigReader): def convert(self): stream = list() - for opt in self._opts: + combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) + opts = Action.substituteRecursiveTags(combinedopts) + if not opts: + raise ValueError('recursive tag definitions unable to be resolved') + for opt, value in opts.iteritems(): if opt == "failregex": - for regex in self._opts[opt].split('\n'): + for regex in value.split('\n'): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self._jailName, "addfailregex", regex]) elif opt == "ignoreregex": - for regex in self._opts[opt].split('\n'): + for regex in value.split('\n'): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self._jailName, "addignoreregex", regex]) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 647b9f86..23b5a4e0 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -308,6 +308,34 @@ class FilterReaderTest(unittest.TestCase): output[-1][-1] = "5" self.assertEqual(sorted(filterReader.convert()), sorted(output)) + + def testFilterReaderSubstitionDefault(self): + output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] + filterReader = FilterReader('substition', "jailname", {}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertEqual(sorted(c), sorted(output)) + + def testFilterReaderSubstitionSet(self): + output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] + filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertEqual(sorted(c), sorted(output)) + + def testFilterReaderSubstitionFail(self): + output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] + filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + self.assertRaises(ValueError, FilterReader.convert, filterReader) + + class JailsReaderTest(LogCaptureTestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/files/filter.d/substition.conf b/fail2ban/tests/files/filter.d/substition.conf new file mode 100644 index 00000000..aaf62eae --- /dev/null +++ b/fail2ban/tests/files/filter.d/substition.conf @@ -0,0 +1,8 @@ + +[Definition] + +failregex = to= fromip= + +[Init] + +honeypot = sweet@example.com From 1b037a6f291d8b2a0345372d5d1b4bcb454488ec Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 31 Dec 2013 19:15:11 +1100 Subject: [PATCH 3/6] DOC: document addition of filter options substitution into failregex/ignoreregex --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 7610c0ff..b2a98046 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,7 +62,8 @@ configuration before relying on it. * Multiline regex for Disconnecting: Too many authentication failures for root [preauth]\nConnection closed by 6X.XXX.XXX.XXX [preauth] * Replacing use of deprecated API (.warning, .assertEqual, etc) - * [..a648cc2] Filters can have options now too + * [..a648cc2] Filters can have options now too which are substituted into + failregex / ignoreregex * [..e019ab7] Multiple instances of the same action are allowed in the same jail -- use actname option to disambiguate. From 58a59833677038b12e1039c3139238ccfc555ac7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 2 Jan 2014 10:03:14 +1100 Subject: [PATCH 4/6] ENH: fix fail2ban-regex for filter arguement substition --- bin/fail2ban-regex | 68 +++++++++++++-------------------- fail2ban/client/configreader.py | 4 ++ 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index fc39c958..ea1ac226 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -41,7 +41,7 @@ except ImportError: journal = None from fail2ban.version import version -from fail2ban.client.configparserinc import SafeConfigParserWithIncludes +from fail2ban.client.filterreader import FilterReader from fail2ban.server.filter import Filter from fail2ban.server.failregex import RegexException @@ -206,8 +206,6 @@ class LineStats(object): class Fail2banRegex(object): - CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} - def __init__(self, opts): self._verbose = opts.verbose self._debuggex = opts.debuggex @@ -257,46 +255,34 @@ class Fail2banRegex(object): assert(regextype in ('fail', 'ignore')) regex = regextype + 'regex' if os.path.isfile(value): - reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) - try: - reader.read(value) - print "Use %11s file : %s" % (regex, value) - # TODO: reuse functionality in client - regex_values = [ - RegexStat(m) - for m in reader.get("Definition", regex).split('\n') - if m != ""] - except NoSectionError: - print "No [Definition] section in %s" % value - return False - except NoOptionError: - print "No %s option in %s" % (regex, value) - return False - except MissingSectionHeaderError: - print "No section headers in %s" % value - return False + print "Use %11s file : %s" % (regex, value) + reader = FilterReader(value, 'fail2ban-regex-jail', {}) + reader.setBaseDir(None) - # Read out and set possible value of maxlines - try: - maxlines = reader.get("Init", "maxlines") - except (NoSectionError, NoOptionError): - # No [Init].maxlines found. - pass + if reader.readexplicit(): + reader.getOptions(None) + readercommands = reader.convert() + regex_values = [ + RegexStat(m[3]) + for m in filter( + lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype, + readercommands)] + # Read out and set possible value of maxlines + for command in readercommands: + if command[2] == "maxlines": + maxlines = int(command[3]) + try: + self.setMaxLines(maxlines) + except ValueError: + print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() + return False + elif command[2] == 'addjournalmatch': + journalmatch = command[3] + self.setJournalMatch(shlex.split(journalmatch)) else: - try: - self.setMaxLines(maxlines) - except ValueError: - print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ - "read from %(value)s" % locals() - return False - # Read out and set possible value for journalmatch - try: - journalmatch = reader.get("Init", "journalmatch") - except (NoSectionError, NoOptionError): - # No [Init].journalmatch found. - pass - else: - self.setJournalMatch(shlex.split(journalmatch)) + print "ERROR: failed to read %s" % value + return False else: print "Use %11s line : %s" % (regex, shortstr(value)) regex_values = [RegexStat(value)] diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index ddbc48db..dbda0608 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -157,6 +157,10 @@ class DefinitionInitConfigReader(ConfigReader): def read(self): return ConfigReader.read(self, self._file) + + # needed for fail2ban-regex that doesn't need fancy directories + def readexplicit(self): + return SafeConfigParserWithIncludes.read(self, self._file) def getOptions(self, pOpts): self._opts = ConfigReader.getOptions( From e6a329210f7478d96d47f5fe878567245c910438 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 2 Jan 2014 10:59:18 +1100 Subject: [PATCH 5/6] correct overprune on imports to filterreader.py --- fail2ban/client/filterreader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index dfdd2ac5..5033d8d3 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -24,6 +24,8 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import logging, os, shlex + from fail2ban.client.configreader import ConfigReader, DefinitionInitConfigReader from fail2ban.server.action import CommandAction From 95add8a1c5cf2e6a91518ed528512038b6fe66e6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 6 Jan 2014 09:55:53 +1100 Subject: [PATCH 6/6] BF: datepattern handling in fail2ban-regex --- bin/fail2ban-regex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index ea1ac226..cfaa4a89 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -280,6 +280,9 @@ class Fail2banRegex(object): elif command[2] == 'addjournalmatch': journalmatch = command[3] self.setJournalMatch(shlex.split(journalmatch)) + elif command[2] == 'datepattern': + datepattern = command[3] + self.setDatePattern(datepattern) else: print "ERROR: failed to read %s" % value return False