From 97f9cfc0b05cd00ada984eed6514b96546fb30a1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 22:56:36 -0400 Subject: [PATCH 01/93] ENH: 'heavydebug' level == 5 for even more debugging in tricky cases I mocked logging library directly -- seems to be Ok. --- common/__init__.py | 5 +++++ fail2ban-testcases | 5 +++-- server/filter.py | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/common/__init__.py b/common/__init__.py index 2b76f4b6..3eae8ee3 100644 --- a/common/__init__.py +++ b/common/__init__.py @@ -23,3 +23,8 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" + +import logging + +# Custom debug level +logging.HEAVYDEBUG = 5 diff --git a/fail2ban-testcases b/fail2ban-testcases index 0bd70259..fc5cc146 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -52,7 +52,7 @@ def get_opt_parser(): p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), default=None, help="Log level for the logger to use during running tests"), Option('-n', "--no-network", action="store_true", @@ -76,7 +76,8 @@ parser = get_opt_parser() logSys = logging.getLogger("fail2ban") # Numerical level of verbosity corresponding to a log "level" -verbosity = {'debug': 3, +verbosity = {'heavydebug': 4, + 'debug': 3, 'info': 2, 'warn': 1, 'error': 1, diff --git a/server/filter.py b/server/filter.py index 90530f92..59e612ed 100644 --- a/server/filter.py +++ b/server/filter.py @@ -291,6 +291,8 @@ class Filter(JailThread): except UnicodeDecodeError: l = line l = l.rstrip('\r\n') + + logSys.log(5, "Working on line %r", l) timeMatch = self.dateDetector.matchTime(l) if timeMatch: # Lets split into time part and log part of the line @@ -355,6 +357,8 @@ class Filter(JailThread): if failRegex.hasMatched(): # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) + logSys.log(7, "Date: %r, message: %r", + timeLine, logLine) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" From ffe381d91cbe92cf505df93648fd322f5673155d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 22:19:10 -0400 Subject: [PATCH 02/93] RF: reworked -regex cmdline tool to use optparse, some unification and enhancement of outputs --- fail2ban-regex | 464 +++++++++++++++++++++++++------------------------ 1 file changed, 237 insertions(+), 227 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index a0a90b05..af38d4d9 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -17,9 +17,17 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +Report bugs to https://github.com/fail2ban/fail2ban/issues +""" __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" import getopt, sys, time, logging, os @@ -37,202 +45,201 @@ from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderErro from server.filter import Filter from server.failregex import RegexException +from optparse import OptionParser, Option + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.regex") -class RegexStat: +def shortstr(s, l=53): + """Return shortened string + """ + if len(s) > l: + return s[:l-3] + '...' + return s + +def pprint_list(l, header=None): + if not len(l): + return + if header: + s = "|- %s\n" % header + else: + s = '' + print s + "| " + "\n| ".join(l) + '\n`-' + +def get_opt_parser(): + # use module docstring for help output + p = OptionParser( + usage="%s [OPTIONS] [IGNOREREGEX]\n" % sys.argv[0] + __doc__ + + """ +LOG: + string a string representing a log line + filename path to a log file (/var/log/auth.log) + +REGEX: + string a string representing a 'failregex' + filename path to a filter file (filter.d/sshd.conf) + +IGNOREREGEX: + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +""", + version="%prog " + version) + + p.add_options([ + Option('-l', "--log-level", type="choice", + dest="log_level", + choices=('debug', 'info', 'warn', 'error', 'fatal'), + default=None, + help="Log level for the Fail2Ban logger to use"), + Option("-v", "--verbose", action='store_true', + help="Be verbose in output"), + Option("--print-all-missed", action='store_true', + help="Either to print all missed lines"), + Option("--print-all-ignored", action='store_true', + help="Either to print all ignored lines"), + + ]) + + return p + + +class RegexStat(object): def __init__(self, failregex): - self.__stats = 0 - self.__failregex = failregex - self.__ipList = list() + self._stats = 0 + self._failregex = failregex + self._ipList = list() def __str__(self): return "%s(%r) %d failed: %s" \ - % (self.__class__, self.__failregex, self.__stats, self.__ipList) + % (self.__class__, self._failregex, self._stats, self._ipList) def inc(self): - self.__stats += 1 + self._stats += 1 def getStats(self): - return self.__stats + return self._stats def getFailRegex(self): - return self.__failregex + return self._failregex def appendIP(self, value): - self.__ipList.extend(value) + self._ipList.extend(value) def getIPList(self): - return self.__ipList + return self._ipList -class Fail2banRegex: +class LineStats(object): + """Just a convenience container for stats + """ + def __init__(self): + self.tested = self.matched = 0 + self.missed_lines = [] + self.ignored_lines = [] + + def __str__(self): + return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self + + @property + def ignored(self): + return len(self.ignored_lines) + + @property + def missed(self): + return self.tested - (self.ignored + self.matched) + + # just for convenient str + def __getitem__(self, key): + return getattr(self, key) + + +class Fail2banRegex(object): + + # ??? test = None CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} - def __init__(self): - self.__filter = Filter(None) - self.__ignoreregex = list() - self.__failregex = list() - self.__verbose = False + def __init__(self, opts): + self._verbose = opts.verbose + self._print_all_missed = opts.print_all_missed + self._print_all_ignored = opts.print_all_ignored + + self._filter = Filter(None) + self._ignoreregex = list() + self._failregex = list() + self._line_stats = LineStats() + # Setup logging logging.getLogger("fail2ban").handlers = [] - self.__hdlr = logging.StreamHandler(Fail2banRegex.test) + self._hdlr = logging.StreamHandler(Fail2banRegex.test) # set a format which is simpler for console use formatter = logging.Formatter("%(message)s") # tell the handler to use this format - self.__hdlr.setFormatter(formatter) - self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self.__hdlr) + self._hdlr.setFormatter(formatter) + self._logging_level = self._verbose and logging.DEBUG or logging.WARN + logging.getLogger("fail2ban").addHandler(self._hdlr) logging.getLogger("fail2ban").setLevel(logging.ERROR) - #@staticmethod - def dispVersion(): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - dispVersion = staticmethod(dispVersion) - #@staticmethod - def dispUsage(): - print "Usage: "+sys.argv[0]+" [OPTIONS] [IGNOREREGEX]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "This tools can test regular expressions for \"fail2ban\"." - print - print "Options:" - print " -h, --help display this help message" - print " -V, --version print the version" - print " -v, --verbose verbose output" - print - print "Log:" - print " string a string representing a log line" - print " filename path to a log file (/var/log/auth.log)" - print - print "Regex:" - print " string a string representing a 'failregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "IgnoreRegex:" - print " string a string representing an 'ignoreregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" - dispUsage = staticmethod(dispUsage) - - def getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) - elif opt[0] in ["-v", "--verbose"]: - self.__verbose = True - - #@staticmethod - def logIsFile(value): - return os.path.isfile(value) - logIsFile = staticmethod(logIsFile) - - def readIgnoreRegex(self, value): + def readRegex(self, value, regextype): + assert(regextype in ('fail', 'ignore')) + regex = regextype + 'regex' if os.path.isfile(value): reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) try: reader.read(value) - print "Use ignoreregex file : " + value - self.__ignoreregex = [RegexStat(m) - for m in reader.get("Definition", "ignoreregex").split('\n')] + print "Use %11s file : %s" % (regex, value) + # TODO: reuse functionality in client + regex_values = [RegexStat(m) + for m in reader.get("Definition", regex).split('\n')] except NoSectionError: - print "No [Definition] section in " + value - print + print "No [Definition] section in %s" % value return False except NoOptionError: - print "No failregex option in " + value - print + print "No %s option in %s" % (regex, value) return False except MissingSectionHeaderError: - print "No section headers in " + value - print + print "No section headers in %s" % value return False else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use ignoreregex line : " + stripReg - self.__ignoreregex = [RegexStat(value)] - return True + print "Use %11s line : %s" % (regex, shortstr(value)) + regex_values = [RegexStat(value)] - def readRegex(self, value): - if os.path.isfile(value): - reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) - try: - reader.read(value) - print "Use regex file : " + value - self.__failregex = [RegexStat(m) - for m in reader.get("Definition", "failregex").split('\n')] - except NoSectionError: - print "No [Definition] section in " + value - print - return False - except NoOptionError: - print "No failregex option in " + value - print - return False - except MissingSectionHeaderError: - print "No section headers in " + value - print - return False - else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use regex line : " + stripReg - self.__failregex = [RegexStat(value)] + setattr(self, "_" + regex, regex_values) return True def testIgnoreRegex(self, line): found = False - for regex in self.__ignoreregex: - logging.getLogger("fail2ban").setLevel(self.__logging_level) + for regex in self._ignoreregex: + logging.getLogger("fail2ban").setLevel(self._logging_level) try: - self.__filter.addIgnoreRegex(regex.getFailRegex()) + self._filter.addIgnoreRegex(regex.getFailRegex()) try: - ret = self.__filter.ignoreLine(line) + ret = self._filter.ignoreLine(line) if ret: + found = True regex.inc() except RegexException, e: print e return False finally: - self.__filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self.__logging_level) + self._filter.delIgnoreRegex(0) + logging.getLogger("fail2ban").setLevel(self._logging_level) + return found def testRegex(self, line): found = False - for regex in self.__ignoreregex: - self.__filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self.__failregex: - logging.getLogger("fail2ban").setLevel(logging.DEBUG) + for regex in self._ignoreregex: + self._filter.addIgnoreRegex(regex.getFailRegex()) + for regex in self._failregex: + # logging.getLogger("fail2ban").setLevel(logging.DEBUG) try: - self.__filter.addFailRegex(regex.getFailRegex()) + self._filter.addFailRegex(regex.getFailRegex()) try: - ret = self.__filter.processLine(line) - if not len(ret) == 0: + ret = self._filter.processLine(line) + if len(ret): if found == True: ret[0].append(True) else: @@ -247,16 +254,47 @@ class Fail2banRegex: print "Sorry, but no found in regex" return False finally: - self.__filter.delFailRegex(0) + self._filter.delFailRegex(0) logging.getLogger("fail2ban").setLevel(logging.CRITICAL) - for regex in self.__ignoreregex: - self.__filter.delIgnoreRegex(0) + for regex in self._ignoreregex: + self._filter.delIgnoreRegex(0) + return found + + + def process(self, test_lines): + + for line in test_lines: + if line.startswith('# ') or not line.strip(): + # skip comment and empty lines + continue + is_ignored = fail2banRegex.testIgnoreRegex(line) + if is_ignored: + self._line_stats.ignored_lines.append(line) + + if fail2banRegex.testRegex(line): + assert(not is_ignored) + self._line_stats.matched += 1 + else: + if not is_ignored: + self._line_stats.missed_lines.append(line) + self._line_stats.tested += 1 + + def printLines(self, ltype): + lstats = self._line_stats + assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) + l = lstats[ltype + '_lines'] + if len(l): + header = "%s line(s):" % (ltype.capitalize(),) + if len(l) < 20 or getattr(self, '_print_all_' + ltype): + pprint_list([x.rstrip() for x in l], header) + else: + print "%s: too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, len(l)) def printStats(self): print print "Results" print "=======" - print def print_failregexes(title, failregexes): # Print title @@ -264,106 +302,78 @@ class Fail2banRegex: for cnt, failregex in enumerate(failregexes): match = failregex.getStats() total += match - if (match or self.__verbose): - out.append("| %d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - print "%s: %d total" % (title, total) - if len(out): - print "|- #) [# of hits] regular expression" - print '\n'.join(out) - print '`-' - print - return total + if (match or self._verbose): + out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - # Print title - total = print_failregexes("Failregex", self.__failregex) - _ = print_failregexes("Ignoreregex", self.__ignoreregex) - - print "Summary" - print "=======" - print - - if total == 0: - print "Sorry, no match" - print - print "Look at the above section 'Running tests' which could contain important" - print "information." - return False - else: - # Print stats - print "Addresses found:" - for cnt, failregex in enumerate(self.__failregex): - if self.__verbose or len(failregex.getIPList()): - print "[%d]" % (cnt+1) + if self._verbose and len(failregex.getIPList()): for ip in failregex.getIPList(): timeTuple = time.localtime(ip[1]) timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - print " %s (%s)%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "") - print + out.append(" %s %s%s" % ( + ip[0], timeString, ip[2] and " (already matched)" or "")) - print "Date template hits:" - for template in self.__filter.dateDetector.getTemplates(): - if self.__verbose or template.getHits(): - print `template.getHits()` + " hit(s): " + template.getName() - print + print "\n%s: %d total" % (title, total) + pprint_list(out, " #) [# of hits] regular expression") + return total - print "Success, the total number of match is " + str(total) - print - print "However, look at the above section 'Running tests' which could contain important" - print "information." - return True + # Print title + total = print_failregexes("Failregex", self._failregex) + _ = print_failregexes("Ignoreregex", self._ignoreregex) + + + print "\nDate template hits:" + out = [] + for template in self._filter.dateDetector.getTemplates(): + if self._verbose or template.getHits(): + out.append("[%d] %s" % (template.getHits(), template.getName())) + pprint_list(out, "[# of hits] date format") + + print "\nLines: %s" % self._line_stats + + self.printLines('ignored') + self.printLines('missed') + + return True if __name__ == "__main__": - fail2banRegex = Fail2banRegex() - # Reads the command line options. - try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] - optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - fail2banRegex.dispUsage() - sys.exit(-1) - # Process command line - fail2banRegex.getCmdLineOptions(optList) + + parser = get_opt_parser() + (opts, args) = parser.parse_args() + + fail2banRegex = Fail2banRegex(opts) # We need 2 or 3 parameters if not len(args) in (2, 3): - fail2banRegex.dispUsage() + sys.stderr.write("ERROR: provide both and .\n\n") + parser.print_help() sys.exit(-1) + + print + print "Running tests" + print "=============" + print + + cmd_log, cmd_regex = args[:2] + + if len(args) == 3: + fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) + + fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log) + print "Use log file : %s" % cmd_log + test_lines = hdlr.readlines() + except IOError, e: + print e + sys.exit(-1) else: - print - print "Running tests" - print "=============" - print + print "Use single line : %s" % shortstr(cmd_log) + test_lines = [ cmd_log ] + print - cmd_log, cmd_regex = args[:2] + fail2banRegex.process(test_lines) - if len(args) == 3: - fail2banRegex.readIgnoreRegex(args[2]) or sys.exit(-1) - - fail2banRegex.readRegex(cmd_regex) or sys.exit(-1) - - if fail2banRegex.logIsFile(cmd_log): - try: - hdlr = open(cmd_log) - print "Use log file : " + cmd_log - print - for line in hdlr: - fail2banRegex.testIgnoreRegex(line) - fail2banRegex.testRegex(line) - except IOError, e: - print e - print - sys.exit(-1) - else: - if len(sys.argv[1]) > 53: - stripLog = cmd_log[0:50] + "..." - else: - stripLog = cmd_log - print "Use single line: " + stripLog - print - fail2banRegex.testIgnoreRegex(cmd_log) - fail2banRegex.testRegex(cmd_log) - - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex.printStats() or sys.exit(-1) From e91419d361c9120896b67c92a1b811a7edf467ef Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 23:01:35 -0400 Subject: [PATCH 03/93] ENH: fail2ban-regex -- add specification of loglevels to enable --- fail2ban-regex | 60 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index af38d4d9..4f5ffd73 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -40,15 +40,16 @@ except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") from common.version import version +from optparse import OptionParser, Option + from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException -from optparse import OptionParser, Option - +from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.regex") +logSys = logging.getLogger("fail2ban") def shortstr(s, l=53): """Return shortened string @@ -88,7 +89,7 @@ IGNOREREGEX: p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), default=None, help="Log level for the Fail2Ban logger to use"), Option("-v", "--verbose", action='store_true', @@ -97,6 +98,10 @@ IGNOREREGEX: help="Either to print all missed lines"), Option("--print-all-ignored", action='store_true', help="Either to print all ignored lines"), + Option("-t", "--log-traceback", action='store_true', + help="Enrich log-messages with compressed tracebacks"), + Option("--full-traceback", action='store_true', + help="Either to make the tracebacks full, not compressed (as by default)"), ]) @@ -156,9 +161,6 @@ class LineStats(object): class Fail2banRegex(object): - # ??? - test = None - CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} def __init__(self, opts): @@ -171,17 +173,6 @@ class Fail2banRegex(object): self._failregex = list() self._line_stats = LineStats() - # Setup logging - logging.getLogger("fail2ban").handlers = [] - self._hdlr = logging.StreamHandler(Fail2banRegex.test) - # set a format which is simpler for console use - formatter = logging.Formatter("%(message)s") - # tell the handler to use this format - self._hdlr.setFormatter(formatter) - self._logging_level = self._verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self._hdlr) - logging.getLogger("fail2ban").setLevel(logging.ERROR) - def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) @@ -213,7 +204,6 @@ class Fail2banRegex(object): def testIgnoreRegex(self, line): found = False for regex in self._ignoreregex: - logging.getLogger("fail2ban").setLevel(self._logging_level) try: self._filter.addIgnoreRegex(regex.getFailRegex()) try: @@ -226,7 +216,6 @@ class Fail2banRegex(object): return False finally: self._filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self._logging_level) return found def testRegex(self, line): @@ -234,7 +223,6 @@ class Fail2banRegex(object): for regex in self._ignoreregex: self._filter.addIgnoreRegex(regex.getFailRegex()) for regex in self._failregex: - # logging.getLogger("fail2ban").setLevel(logging.DEBUG) try: self._filter.addFailRegex(regex.getFailRegex()) try: @@ -255,7 +243,6 @@ class Fail2banRegex(object): return False finally: self._filter.delFailRegex(0) - logging.getLogger("fail2ban").setLevel(logging.CRITICAL) for regex in self._ignoreregex: self._filter.delIgnoreRegex(0) return found @@ -349,6 +336,35 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) + # TODO: taken from -testcases -- move common functionality somewhere + if opts.log_level is not None: # pragma: no cover + # so we had explicit settings + logSys.setLevel(getattr(logging, opts.log_level.upper())) + else: # pragma: no cover + # suppress the logging but it would leave unittests' progress dots + # ticking, unless like with '-l fatal' which would be silent + # unless error occurs + logSys.setLevel(getattr(logging, 'FATAL')) + + # Add the default logging handler + stdout = logging.StreamHandler(sys.stdout) + + fmt = 'D: %(message)s' + + if opts.log_traceback: + Formatter = FormatterWithTraceBack + fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt + else: + Formatter = logging.Formatter + + # Custom log format for the verbose tests runs + if opts.verbose > 1: # pragma: no cover + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) + else: # pragma: no cover + # just prefix with the space + stdout.setFormatter(Formatter(fmt)) + logSys.addHandler(stdout) + print print "Running tests" print "=============" From 77044fce35be6e7e1d6310521227f6a25ea61ac9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Jun 2013 23:21:48 -0400 Subject: [PATCH 04/93] DOC: Changelog for fail2ban-regex RF --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1ea46aed..316d20f2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - Enhancements: Daniel Black * config/filter.d/asterisk.conf -- more stringent anchoring + Yaroslav Halchenko + * fail2ban-regex -- refactored to provide more details (missing and + ignored lines, control over logging, etc) while maintaining look&feel. ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From d441d61a1e6d982d408823d807028c8233abbb59 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 15 Jun 2013 12:34:16 +1000 Subject: [PATCH 05/93] TST/ENH: Improve regex around exim rejected by local_scan now has test cases. Unrouteable address error messages now normalised after looking into exim code. --- config/filter.d/exim.conf | 7 ++++++- testcases/files/logs/exim | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index d335c9fc..da975d61 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -13,7 +13,12 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^ H=\S+ \(\S+\) \[\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$ + +# From exim source code: ./src/receive.c:add_host_info_for_log +host_info = H=\S+ (\(\S+\) )?\[\] (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )? + +failregex = ^ %(host_info)ssender verify fail for <\S+>: Unrouteable address\s*$ + ^ \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ # Option: ignoreregex diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index d86b6455..d7f0e58b 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -4,3 +4,9 @@ 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address +# http://forum.lissyara.su/viewtopic.php?f=20&t=2985 +2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html +2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html +2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 From b8cfda68b8dfda32648db621cd256d334eca0365 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 16 Jun 2013 00:19:37 +1000 Subject: [PATCH 06/93] ENH: new exim filter regexs. Also note a begining PID in this format. Thanks to ftoppi for the log entries --- ChangeLog | 2 +- THANKS | 1 + config/filter.d/exim.conf | 13 +++++++++---- testcases/files/logs/exim | 8 ++++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index f0df7ddc..842932fc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,7 +19,7 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening and extra failure examples in sample logs - Daniel Black & Georgiy Mernov + Daniel Black & Georgiy Mernov & ftoppi * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs diff --git a/THANKS b/THANKS index 47c3e999..f8b18daa 100644 --- a/THANKS +++ b/THANKS @@ -18,6 +18,7 @@ Daniel Black David Nutter Eric Gerbier Enrico Labedzki +ftoppi Georgiy Mernov Guillaume Delvit Hanno 'Rince' Wagner diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index da975d61..62184fbf 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -15,11 +15,16 @@ # # From exim source code: ./src/receive.c:add_host_info_for_log -host_info = H=\S+ (\(\S+\) )?\[\] (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )? +host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )? +pid = ( \[\d+\])? -failregex = ^ %(host_info)ssender verify fail for <\S+>: Unrouteable address\s*$ - ^ \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ - ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ +failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: Unrouteable address\s*$ + ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ + ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (rejected found in dnsbl \S+|relay not permitted)\s*$ + ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ + ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ + ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index d7f0e58b..ebcb9726 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -10,3 +10,11 @@ 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory # http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html 2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 +# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 +2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org +2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted +2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected +2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL). +2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n" +2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n" +2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET") From 25c3bbfc2fead08c54891f293f5522d61199a62e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 16 Jun 2013 00:25:24 +1000 Subject: [PATCH 07/93] DOC: credits/blame to me for changes to exim --- config/filter.d/exim.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 62184fbf..252da07a 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -1,7 +1,7 @@ # Fail2Ban configuration file # # Author: Cyril Jaquier -# +# Daniel Black (rewrote with strong regexs) # [Definition] From 2974cac40cc7709909fcb11a510f164c88e7db46 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 19 Jun 2013 22:10:18 -0400 Subject: [PATCH 08/93] RF: log all logging output from fail2ban-client to stderr. Close #264 otherwise it 1. 'interferes' with meaninful output of the client 2. if ERROR is logged it better go to stderr and separating ERROR from other levels is not that transparent with python's logging --- ChangeLog | 3 ++- fail2ban-client | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index a6bc4e21..f2425d2f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,7 +24,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released sample logs Yaroslav Halchenko * fail2ban-regex -- refactored to provide more details (missing and - ignored lines, control over logging, etc) while maintaining look&feel. + ignored lines, control over logging, etc) while maintaining look&feel + * fail2ban-client -- log to standard error ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/fail2ban-client b/fail2ban-client index de8519f4..57235d79 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -336,13 +336,13 @@ class Fail2banClient: logSys.setLevel(logging.INFO) else: logSys.setLevel(logging.DEBUG) - # Add the default logging handler - stdout = logging.StreamHandler(sys.stdout) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) # set a format which is simpler for console use formatter = logging.Formatter('%(levelname)-6s %(message)s') # tell the handler to use this format - stdout.setFormatter(formatter) - logSys.addHandler(stdout) + logout.setFormatter(formatter) + logSys.addHandler(logout) # Set the configuration path self.__configurator.setBaseDir(self.__conf["conf"]) From b6be8b8243273bbe6549a1f1057e8633e767c314 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 21 Jun 2013 11:09:16 -0400 Subject: [PATCH 09/93] ENH/RF: remove __readJailConfig in favor of __readConfig + catch/error exceptions while reading the configuration --- fail2ban-client | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/fail2ban-client b/fail2ban-client index 57235d79..be48a613 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -229,7 +229,7 @@ class Fail2banClient: elif len(cmd) == 2 and cmd[0] == "reload": if self.__ping(): jail = cmd[1] - ret = self.__readJailConfig(jail) + ret = self.__readConfig(jail) # Do not continue if configuration is not 100% valid if not ret: return False @@ -394,19 +394,18 @@ class Fail2banClient: return False return self.__processCommand(args) - def __readConfig(self): + def __readConfig(self, jail=None): # Read the configuration - self.__configurator.readAll() - ret = self.__configurator.getOptions() - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() - return ret - - def __readJailConfig(self, jail): - self.__configurator.readAll() - ret = self.__configurator.getOptions(jail) - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() + # TODO: get away from stew of return codes and exception + # handling -- handle via exceptions + try: + self.__configurator.readAll() + ret = self.__configurator.getOptions(jail) + self.__configurator.convertToProtocol() + self.__stream = self.__configurator.getConfigStream() + except Exception, e: + logSys.error("Failed during configuration: %s" % e) + ret = False return ret #@staticmethod From 27947407bc7910f0f50972113218ebc73c4a22c7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 21 Jun 2013 11:08:27 -0400 Subject: [PATCH 10/93] ENH: raise an exception if not a single file was found for the jail. Close #63 --- client/jailreader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/jailreader.py b/client/jailreader.py index f8757e26..ef444125 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -107,12 +107,17 @@ class JailReader(ConfigReader): stream = [] for opt in self.__opts: if opt == "logpath": + found_files = 0 for path in self.__opts[opt].split("\n"): pathList = glob.glob(path) if len(pathList) == 0: - logSys.error("No file found for " + path) + logSys.error("No file(s) found for glob %s" % path) for p in pathList: + found_files += 1 stream.append(["set", self.__name, "addlogpath", p]) + if not found_files: + raise ValueError( + "Have not found any log file for %s jail" % self.__name) elif opt == "backend": backend = self.__opts[opt] elif opt == "maxretry": From 61f81c6b2f17576e970967a918997af55bc2b829 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 21 Jun 2013 11:12:27 -0400 Subject: [PATCH 11/93] Changelog entries with close statements for recent changes --- ChangeLog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f2425d2f..0dd7a073 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,7 +25,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Yaroslav Halchenko * fail2ban-regex -- refactored to provide more details (missing and ignored lines, control over logging, etc) while maintaining look&feel - * fail2ban-client -- log to standard error + * fail2ban-client -- log to standard error. Closes gh-264 + * Fail to configure if not a single log file was found for an + enabled jail. Closes gh-63 ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From 057f0ad1357273f027b944e046d66e486c2f197f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 21 Jun 2013 12:44:37 -0400 Subject: [PATCH 12/93] ENH: allow_no_files option for jail's convert to allow testing of stock jail.conf --- client/jailreader.py | 13 +++++++++++-- client/jailsreader.py | 13 +++++++++++-- testcases/clientreadertestcase.py | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/client/jailreader.py b/client/jailreader.py index ef444125..1a2be432 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -103,7 +103,16 @@ class JailReader(ConfigReader): logSys.warn("No actions were defined for %s" % self.__name) return True - def convert(self): + def convert(self, allow_no_files=False): + """Convert read before __opts to the commands stream + + Parameters + ---------- + allow_missing : bool + Either to allow log files to be missing entirely. Primarily is + used for testing + """ + stream = [] for opt in self.__opts: if opt == "logpath": @@ -115,7 +124,7 @@ class JailReader(ConfigReader): for p in pathList: found_files += 1 stream.append(["set", self.__name, "addlogpath", p]) - if not found_files: + if not (found_files or allow_no_files): raise ValueError( "Have not found any log file for %s jail" % self.__name) elif opt == "backend": diff --git a/client/jailsreader.py b/client/jailsreader.py index 098b525d..408a40bf 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -79,14 +79,23 @@ class JailsReader(ConfigReader): return False return True - def convert(self): + def convert(self, allow_no_files=False): + """Convert read before __opts and jails to the commands stream + + Parameters + ---------- + allow_missing : bool + Either to allow log files to be missing entirely. Primarily is + used for testing + """ + stream = list() for opt in self.__opts: if opt == "": stream.append([]) # Convert jails for jail in self.__jails: - stream.extend(jail.convert()) + stream.extend(jail.convert(allow_no_files=allow_no_files)) # Start jails for jail in self.__jails: stream.append(["start", jail.getName()]) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index a8c60bf8..948431b3 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -132,7 +132,7 @@ class JailsReaderTest(unittest.TestCase): jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine - comm_commands = jails.convert() + comm_commands = jails.convert(allow_no_files=True) # by default we have lots of jails ;) self.assertTrue(len(comm_commands)) From fa7a105483e6653d45830ae0336c84daba6899fc Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 27 Jun 2013 09:16:14 +1000 Subject: [PATCH 13/93] ENH: filter.d/asterisk - consolidate log prefix regex and add a few fail messages --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/asterisk.conf | 29 ++++++++++++++++------------- testcases/files/logs/asterisk | 6 ++++++ 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index a6bc4e21..44744a76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Daniel Black & Georgiy Mernov * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs + Daniel Black & Sebastian Arcus + * filter.d/asterisk -- more regexes Yaroslav Halchenko * fail2ban-regex -- refactored to provide more details (missing and ignored lines, control over logging, etc) while maintaining look&feel. diff --git a/THANKS b/THANKS index 47c3e999..c549d77f 100644 --- a/THANKS +++ b/THANKS @@ -40,6 +40,7 @@ Raphaël Marichez René Berber Robert Edeker Russell Odom +Sebastian Arcus Sireyessire silviogarbes Stephen Gildea diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index a8f65e09..6d351f8b 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -17,19 +17,22 @@ before = common.conf # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ - ^\[\]\s*NOTICE%(__pid_re)s\[\S+\] \S+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed to authenticate as '[^']*'$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: No registration for peer '[^']*' \(from \)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Failed to authenticate user [^@]+@\S*$ - ^\[\]\s*SECURITY%(__pid_re)s \S+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ +log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d* + +failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ + ^%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^%(log_prefix)s Host failed to authenticate as '[^']*'$ + ^%(log_prefix)s No registration for peer '[^']*' \(from \)$ + ^%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^%(log_prefix)s Failed to authenticate user [^@]+@\S*$ + ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S* + ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index 45b69304..e2fe3b6e 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -13,3 +13,9 @@ [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found [2013-05-13 07:10:53] SECURITY[1204] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="1368439853-500975",Severity="Error",Service="SIP",EventVersion="1",AccountID="00972599580679",SessionID="0x7f8ecc0421f8",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/1.2.3.4/5070" [2013-06-10 18:15:03] NOTICE[2723] chan_sip.c: Registration from '"100"' failed for '1.2.3.4' - Not a local domain +# http://forum.4psa.com/showthread.php?t=6601 +[2009-12-22 16:35:24] NOTICE[6163] chan_sip.c: Sending fake auth rejection for device ;tag=e3793a95e1acbc69o +# http://www.freepbx.org/forum/general-help/fake-auth-rejection +[2009-12-22 16:35:24] NOTICE[1570][C-00000086] chan_sip.c: Sending fake auth rejection for device 1022;tag=5d8b6f92 +# http://www.spinics.net/lists/asterisk/msg127381.html +[2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 From 6d331bcbea049b31a146f2196aeb33bfdccbe30f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 27 Jun 2013 11:43:27 -0400 Subject: [PATCH 14/93] BF: make colon after [daemon] optional. Close #267 --- ChangeLog | 3 ++- config/filter.d/common.conf | 2 +- testcases/files/logs/sshd | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index a3ac95ce..45c6f343 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,7 +11,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released ----------- - Fixes: - + Yaroslav Halchenko + * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index aafeb5ca..d44a6325 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -33,7 +33,7 @@ __daemon_extra_re = (?:\[ID \d+ \S+\]) # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] -__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:) +__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?) # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 5dab1606..1943fb37 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -46,3 +46,6 @@ Apr 29 15:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication error Apr 29 16:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.101 via 192.168.1.201 Apr 29 17:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.102 Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.103 + +#11 https://github.com/fail2ban/fail2ban/issues/267 There might be no colon after [daemon] +Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers From 1b170b2aef828899ad6596ada0c379c6e07a620e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 28 Jun 2013 09:34:12 -0400 Subject: [PATCH 15/93] BF: support apache 2.4 more detailed error log format. Close #268 --- ChangeLog | 2 ++ config/filter.d/apache-common.conf | 6 +++++- testcases/files/logs/apache-nohome | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 testcases/files/logs/apache-nohome diff --git a/ChangeLog b/ChangeLog index 45c6f343..e5c38eb1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - Fixes: Yaroslav Halchenko * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 + * filter.d/apache-common.conf -- support apache 2.4 more detailed error + log format. Closes gh-268 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index c3829e2f..cc35ae5f 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -14,4 +14,8 @@ after = apache-common.local [DEFAULT] # Common prefix for [error] apache messages which also would include -_apache_error_client = \[[^]]+\] \[error\] \[client \] +# Depending on the version it could be +# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] +# 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652] +# Reference: https://github.com/fail2ban/fail2ban/issues/268 +_apache_error_client = \[[^]]+\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\]( \S+:)? diff --git a/testcases/files/logs/apache-nohome b/testcases/files/logs/apache-nohome new file mode 100644 index 00000000..e8ef56bc --- /dev/null +++ b/testcases/files/logs/apache-nohome @@ -0,0 +1,4 @@ +# Apache 2.2 +[Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] File does not exist: /xxx/~ +# Apache 2.4 +[Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 192.0.2.12:46652] AH00128: File does not exist: /xxx/~ From 0086a7edabaffc0524ed8dd02e679c9cdcd577fb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 29 Jun 2013 11:30:37 +1000 Subject: [PATCH 16/93] ENH: missed a $ --- config/filter.d/asterisk.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 6d351f8b..5e775430 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -31,7 +31,7 @@ failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' ^%(log_prefix)s No registration for peer '[^']*' \(from \)$ ^%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ ^%(log_prefix)s Failed to authenticate user [^@]+@\S*$ - ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S* + ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S*$ ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ # Option: ignoreregex From ec0680d73ad6eb8cc136ac5493be9a636b1ee519 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Jun 2013 19:17:22 +0100 Subject: [PATCH 17/93] BF: fail2ban-regex when using ignoreregex file with empty regex --- fail2ban-regex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 4f5ffd73..78716a43 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -183,8 +183,10 @@ class Fail2banRegex(object): reader.read(value) print "Use %11s file : %s" % (regex, value) # TODO: reuse functionality in client - regex_values = [RegexStat(m) - for m in reader.get("Definition", regex).split('\n')] + regex_values = [ + RegexStat(m) + for m in reader.get("Definition", regex).split('\n') + if m != ""] except NoSectionError: print "No [Definition] section in %s" % value return False From b670b5c792bcdd6d49e25a84b52799a86b2abcfe Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Jun 2013 20:17:42 +0100 Subject: [PATCH 18/93] TST: Change depreciated assertEquals to assertEqual in some tests --- testcases/actiontestcase.py | 8 ++++---- testcases/clientreadertestcase.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testcases/actiontestcase.py b/testcases/actiontestcase.py index 9a7776ac..1aa9f24e 100644 --- a/testcases/actiontestcase.py +++ b/testcases/actiontestcase.py @@ -69,11 +69,11 @@ class ExecuteAction(unittest.TestCase): self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': ''})) self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) # missing tags are ok - self.assertEquals(Action.substituteRecursiveTags({'A': ''}), {'A': ''}) - self.assertEquals(Action.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) - self.assertEquals(Action.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(Action.substituteRecursiveTags({'A': ''}), {'A': ''}) + self.assertEqual(Action.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) + self.assertEqual(Action.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # rest is just cool - self.assertEquals(Action.substituteRecursiveTags(aInfo), + self.assertEqual(Action.substituteRecursiveTags(aInfo), { 'HOST': "192.0.2.0", 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 948431b3..695b3877 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -108,7 +108,7 @@ class JailReaderTest(unittest.TestCase): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] result = JailReader.splitAction(action) - self.assertEquals(expected, result) + self.assertEqual(expected, result) class JailsReaderTest(unittest.TestCase): From 3b76fc79f9a6269aca915fc33ea15fcef1709ffb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 1 Jul 2013 21:12:51 +1000 Subject: [PATCH 19/93] BF: fix dovecot filter for when no TLS is enabled on pop/imap --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/dovecot.conf | 2 +- testcases/files/logs/dovecot | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 45c6f343..c11215a7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - Fixes: Yaroslav Halchenko * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 + Daniel Black & Мернов Георгий + * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in , - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/THANKS b/THANKS index f8b18daa..eac3c6f0 100644 --- a/THANKS +++ b/THANKS @@ -33,6 +33,7 @@ Mark Edgington Markus Hoffmann Marvin Rouge mEDI +Мернов Георгий Michael C. Haller Michael Hanselmann NickMunger diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index dd4c35ba..e3702fcf 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -17,7 +17,7 @@ _daemon = dovecot(-auth)? # Values: TEXT # failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3},( TLS( handshaking)?(: Disconnected)?)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(?:\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index f904a8fe..01df0af3 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -12,4 +12,5 @@ Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attem Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected +Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 From 72f9e6a51ecf12abb92d6d14349f3558c5e60c7c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 1 Jul 2013 21:50:35 +1000 Subject: [PATCH 20/93] ENH/TST: more samples and rejection types for sender verify fail and rejected RCPT --- ChangeLog | 2 +- config/filter.d/exim.conf | 4 ++-- testcases/files/logs/exim | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c11215a7..549eb7f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,7 +22,7 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening and extra failure examples in sample logs - Daniel Black & Georgiy Mernov & ftoppi + Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs Yaroslav Halchenko diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 252da07a..fc744dd3 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -18,10 +18,10 @@ host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )? pid = ( \[\d+\])? -failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: Unrouteable address\s*$ +failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (rejected found in dnsbl \S+|relay not permitted)\s*$ + ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (rejected found in dnsbl \S+|relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index ebcb9726..792f949e 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -18,3 +18,9 @@ 2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n" 2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n" 2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET") +2013-06-02 15:06:18 H=(VM-WIN2K3-1562) [46.20.35.114] sender verify fail for : Unknown user +2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for : all relevant MX records point to non-existent hosts +2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F= rejected RCPT : Sender verify failed +2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F= rejected RCPT : Unknown user +#2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 + From 9757e1df2b84bd40a42d52eb4ede6890092998a9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 1 Jul 2013 21:53:05 +1000 Subject: [PATCH 21/93] ENH: make groupings non-capturing --- config/filter.d/exim.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index fc744dd3..9e8491b7 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -20,11 +20,11 @@ pid = ( \[\d+\])? failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ - ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (rejected found in dnsbl \S+|relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s login authenticator failed for (?:\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:rejected found in dnsbl \S+|relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ - ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ - ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ + ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ + ^%(pid)s SMTP call from \S+ \[\](:\d+)? (?:I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From ca996ace5e9bed218223da7fd96eff6606e64c04 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 1 Jul 2013 21:56:02 +1000 Subject: [PATCH 22/93] ENH: remove temporary failures from local_scan in line with comments in gh-258 --- config/filter.d/exim.conf | 2 +- testcases/files/logs/exim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 9e8491b7..49a7d2c4 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -19,7 +19,7 @@ host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (?:I=\[\S+\]:\d+ )?(?:U=\S pid = ( \[\d+\])? failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ - ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$ + ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$ ^%(pid)s login authenticator failed for (?:\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$ ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:rejected found in dnsbl \S+|relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 792f949e..2a59ecdc 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -5,9 +5,9 @@ 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address # http://forum.lissyara.su/viewtopic.php?f=20&t=2985 -2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +#2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html -2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +#2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory # http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html 2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 # https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 From c7d64c3c7f9dee7b300f97e6c6f444b7ca6e8c10 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 1 Jul 2013 21:58:03 +1000 Subject: [PATCH 23/93] TST: url reference fix --- testcases/files/logs/exim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 2a59ecdc..96b901ab 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -4,7 +4,7 @@ 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address -# http://forum.lissyara.su/viewtopic.php?f=20&t=2985 +# http://forum.lissyara.su/viewtopic.php?f=20&t=29857 #2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html #2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory From 4777cfd4e7d45c13cf6f7fcfcf2aadaed323caa6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 2 Jul 2013 20:03:16 +1000 Subject: [PATCH 24/93] ENH: split out exim-spam into speparate filter --- config/filter.d/exim-common.conf | 17 +++++++++++++++++ config/filter.d/exim-spam.conf | 29 +++++++++++++++++++++++++++++ config/filter.d/exim.conf | 31 +++++++++++++++---------------- testcases/files/logs/exim | 6 +++--- 4 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 config/filter.d/exim-common.conf create mode 100644 config/filter.d/exim-spam.conf diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf new file mode 100644 index 00000000..79d6ffbb --- /dev/null +++ b/config/filter.d/exim-common.conf @@ -0,0 +1,17 @@ +# Fail2Ban configuration file for exim +# +# Author: Daniel Black +# + +[INCLUDES] + +# Load customizations if any available +# +after = exim-common.local + +[Definition] + +# From exim source code: ./src/receive.c:add_host_info_for_log +host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (I=\[\S+\]:\d+ )?(U=\S+ )?(P=e?smtp )? +pid = ( \[\d+\])? + diff --git a/config/filter.d/exim-spam.conf b/config/filter.d/exim-spam.conf new file mode 100644 index 00000000..55a6f5dd --- /dev/null +++ b/config/filter.d/exim-spam.conf @@ -0,0 +1,29 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Daniel Black (rewrote with strong regexs) +# + + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# exim-common.local +before = exim-common.conf + + +[Definition] + +# Option: failregex +# Notes.: This includes the spam rejection messages of exim. +# Note the %(host_info) defination contains a match + +failregex = ^%(pid)s \S+ F=(<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$ + ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$ + ^%(pid)s \S+ %(host_info)sF=(<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 49a7d2c4..63b0fa1d 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -4,27 +4,26 @@ # Daniel Black (rewrote with strong regexs) # + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# exim-common.local +before = exim-common.conf + + [Definition] # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# Values: TEXT -# - -# From exim source code: ./src/receive.c:add_host_info_for_log -host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )? -pid = ( \[\d+\])? +# Notes.: This includes the rejection messages of exim. For spam and filter +# related bans use the exim-spam.conf +# Note the %(host_info) defination contains a match failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ - ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$ - ^%(pid)s login authenticator failed for (?:\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:rejected found in dnsbl \S+|relay not permitted|Sender verify failed|Unknown user)\s*$ - ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ - ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ - ^%(pid)s SMTP call from \S+ \[\](:\d+)? (?:I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ + ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ + ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 96b901ab..95ca621f 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -5,9 +5,9 @@ 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address # http://forum.lissyara.su/viewtopic.php?f=20&t=29857 -#2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html -#2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory # http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html 2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 # https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 @@ -22,5 +22,5 @@ 2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for : all relevant MX records point to non-existent hosts 2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F= rejected RCPT : Sender verify failed 2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F= rejected RCPT : Unknown user -#2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 +2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 From aebd24ec5485dacc5146728fa22387340b51099e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 2 Jul 2013 20:09:27 +1000 Subject: [PATCH 25/93] BF: replace with ed so its cross platform, fixes permission problem gh-266, and Yaroslav doesn't revert to perl --- ChangeLog | 2 ++ config/action.d/hostsdeny.conf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index e5c38eb1..51835de4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 * filter.d/apache-common.conf -- support apache 2.4 more detailed error log format. Closes gh-268 + Daniel Black + * action.d/hostsdeny -- ensure permissions are fixed - gh-266 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/config/action.d/hostsdeny.conf b/config/action.d/hostsdeny.conf index 50a4545c..36e34948 100644 --- a/config/action.d/hostsdeny.conf +++ b/config/action.d/hostsdeny.conf @@ -39,7 +39,7 @@ actionban = IP= && # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = IP= && sed /ALL:\ $IP/d > .new && mv .new +actionunban = echo "/ALL: $/
d
w
q" | ed [Init] From e6823149a1d4916a80594970fe8f7f7845310e88 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 2 Jul 2013 20:16:43 +1000 Subject: [PATCH 26/93] ENH: remove non-capturing groups for readibility --- config/filter.d/dovecot.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index e3702fcf..f111859f 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -16,8 +16,8 @@ _daemon = dovecot(-auth)? # first regex is essentially a copy of pam-generic.conf # Values: TEXT # -failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(?:\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?\s*$ +failregex = ^%(__prefix_line)s(pam_unix(\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From da594075f3f59476ef055a6387634468fa839036 Mon Sep 17 00:00:00 2001 From: Alexander Dietrich Date: Tue, 2 Jul 2013 20:30:41 +0200 Subject: [PATCH 27/93] Move sendmail settings to common file, make sender name configurable --- config/action.d/sendmail-buffered.conf | 18 +++++++----------- config/action.d/sendmail-common.conf | 23 +++++++++++++++++++++++ config/action.d/sendmail-whois-lines.conf | 18 +++++++----------- config/action.d/sendmail-whois.conf | 18 +++++++----------- config/action.d/sendmail.conf | 18 +++++++----------- 5 files changed, 51 insertions(+), 44 deletions(-) create mode 100644 config/action.d/sendmail-common.conf diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf index bec1e91c..f5ca6c10 100644 --- a/config/action.d/sendmail-buffered.conf +++ b/config/action.d/sendmail-buffered.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -11,7 +15,7 @@ # Values: CMD # actionstart = printf %%b "Subject: [Fail2Ban] : started - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = if [ -f ]; then printf %%b "Subject: [Fail2Ban] : summary - From: Fail2Ban <> + From: <> To: \n Hi,\n These hosts have been banned by Fail2Ban.\n @@ -58,7 +62,7 @@ actionban = printf %%b "`date`: ( failures)\n" >> LINE=$( wc -l | awk '{ print $1 }' ) if [ $LINE -ge ]; then printf %%b "Subject: [Fail2Ban] : summary - From: Fail2Ban <> + From: <> To: \n Hi,\n These hosts have been banned by Fail2Ban.\n @@ -82,14 +86,6 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - # Default number of lines that are buffered # lines = 5 diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf new file mode 100644 index 00000000..e2820470 --- /dev/null +++ b/config/action.d/sendmail-common.conf @@ -0,0 +1,23 @@ +# Fail2Ban configuration file +# +# Common settings for sendmail actions +# +# Users can override the defaults in sendmail-common.local + +[INCLUDES] + +after = sendmail-common.local + +[Init] + +# Recipient mail address +# +dest = root + +# Sender mail address +# +sender = fail2ban + +# Sender display name +# +sendername = Fail2Ban diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index bc5074c6..2cb27bd2 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -72,14 +76,6 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - # Path to the log files which contain relevant lines for the abuser IP # logpath = /dev/null diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index 0d1fd97e..b111e19f 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -70,11 +74,3 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index 8054050d..55d388fc 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -68,11 +72,3 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - From 8f3671bc94c604ae33a7592d0e72220cbc00dd52 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 17:10:00 -0400 Subject: [PATCH 28/93] BF: figure out minimal sleep time needed for mtime changes to get detected. Close #223, and probably #103 --- testcases/filtertestcase.py | 22 +++++----------- testcases/utils.py | 50 ++++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 1657f694..2381c942 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -39,6 +39,8 @@ from server.failmanager import FailManagerEmpty # Useful helpers # +from utils import mtimesleep + # yoh: per Steven Hiscocks's insight while troubleshooting # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 # adding a sufficiently large buffer might help to guarantee that @@ -68,18 +70,6 @@ def _killfile(f, name): _killfile(None, name + '.bak') -def _sleep_4_poll(): - """PollFilter relies on file timestamps - so we might need to - sleep to guarantee that they differ - """ - if sys.version_info[:2] <= (2,4): - # on old Python st_mtime is int, so we should give - # at least 1 sec so polling filter could detect - # the change - time.sleep(1.) - else: - time.sleep(0.1) - def _assert_equal_entries(utest, found, output, count=None): """Little helper to unify comparisons with the target entries @@ -237,14 +227,14 @@ class LogFileMonitor(unittest.TestCase): # but not any longer self.assertTrue(self.notModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() # to guarantee freshier mtime + mtimesleep() # to guarantee freshier mtime for i in range(4): # few changes # unless we write into it self.file.write("line%d\n" % i) self.file.flush() self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() # to guarantee freshier mtime + mtimesleep() # to guarantee freshier mtime os.rename(self.name, self.name + '.old') # we are not signaling as modified whenever # it gets away @@ -252,7 +242,7 @@ class LogFileMonitor(unittest.TestCase): f = open(self.name, 'a') self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() + mtimesleep() f.write("line%d\n" % i) f.flush() self.assertTrue(self.isModified()) @@ -398,7 +388,7 @@ def get_monitor_failures_testcase(Filter_): # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - _sleep_4_poll() + mtimesleep() def isEmpty(self, delay=0.4): # shorter wait time for not modified status diff --git a/testcases/utils.py b/testcases/utils.py index 87aab915..b8d467bf 100644 --- a/testcases/utils.py +++ b/testcases/utils.py @@ -22,7 +22,7 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback +import logging, os, re, tempfile, sys, time, traceback from os.path import basename, dirname # @@ -100,3 +100,51 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + + +class MTimeSleep(object): + """Sleep minimal duration needed to resolve changes in mtime of files in TMPDIR + + mtime resolution depends on Python version AND underlying filesystem + """ + def __init__(self): + self._sleep = None + + @staticmethod + def _get_good_sleep(): + times = [1., 2., 5., 10.] + # we know that older Pythons simply have no ability to resolve + # at < sec level. + if sys.version_info[:2] > (2, 4): + times = [0.01, 0.1] + times + ffid, name = tempfile.mkstemp() + tfile = os.fdopen(ffid, 'w') + + for stime in times: + prev_stat, dt = "", 0. + # needs to be done 3 times (not clear why) + for i in xrange(3): + stat2 = os.stat(name) + if prev_stat: + dt = (stat2.st_mtime - prev_stat.st_mtime) + prev_stat = stat2 + tfile.write("LOAD\n") + tfile.flush() + time.sleep(stime) + if dt: + break + if not dt: + import logging + logSys = logging.getLogger("fail2ban.tests") + #from warnings import warn + logSys.warn("Could not deduce appropriate sleep time for tests. " + "Maximal tested one of %f sec will be used." % stime) + os.unlink(name) + return stime + + def __call__(self): + if self._sleep is None: + self._sleep = self._get_good_sleep() + time.sleep(self._sleep) + +mtimesleep = MTimeSleep() From 591590860ac3077d9ec24af3c0f998df15820541 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 17:11:24 -0400 Subject: [PATCH 29/93] BF: setSleepTime -- would barf since value is not str (wasn't used/tested) --- server/jailthread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/jailthread.py b/server/jailthread.py index 11d1a82b..ca6effae 100644 --- a/server/jailthread.py +++ b/server/jailthread.py @@ -56,7 +56,7 @@ class JailThread(Thread): def setSleepTime(self, value): self.__sleepTime = value - logSys.info("Set sleeptime = " + value) + logSys.info("Set sleeptime %s" % value) ## # Get the time that the thread sleeps. From e9c8a51ce492d2ee225e514395642f50ae8a64d6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 17:26:41 -0400 Subject: [PATCH 30/93] ENH: further tighten up detection of mtimesleep duration + log what was assigned --- testcases/utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/testcases/utils.py b/testcases/utils.py index b8d467bf..6107516b 100644 --- a/testcases/utils.py +++ b/testcases/utils.py @@ -112,11 +112,12 @@ class MTimeSleep(object): @staticmethod def _get_good_sleep(): - times = [1., 2., 5., 10.] + logSys = logging.getLogger("fail2ban.tests") + times = [1.5, 2., 5., 10.] # we know that older Pythons simply have no ability to resolve # at < sec level. if sys.version_info[:2] > (2, 4): - times = [0.01, 0.1] + times + times = [0.1] + times ffid, name = tempfile.mkstemp() tfile = os.fdopen(ffid, 'w') @@ -131,14 +132,19 @@ class MTimeSleep(object): tfile.write("LOAD\n") tfile.flush() time.sleep(stime) - if dt: - break + + # check dt but also verify that we are not getting 'quick' + # stime simply by chance of catching second increment + if dt and \ + not (stime < 1 and int(stat2.st_mtime) == stat2.st_mtime): + break if not dt: - import logging - logSys = logging.getLogger("fail2ban.tests") #from warnings import warn logSys.warn("Could not deduce appropriate sleep time for tests. " "Maximal tested one of %f sec will be used." % stime) + else: + logSys.debug("It was needed a sleep of %f to detect dt=%f mtime change" + % (stime, dt)) os.unlink(name) return stime From d6dece490043671604ddf9710590e294fd2c9095 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 3 Jul 2013 07:42:47 +1000 Subject: [PATCH 31/93] ENH: Split log and provide jail examples --- config/jail.conf | 11 +++++++++++ testcases/files/logs/exim | 7 ------- testcases/files/logs/exim-spam | 12 ++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 testcases/files/logs/exim-spam diff --git a/config/jail.conf b/config/jail.conf index c999cc7b..b3b93e53 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -408,3 +408,14 @@ filter = 3proxy action = iptables-multiport[name=3proxy, port=3128, protocol=tcp] logpath = /var/log/3proxy.log +[exim] +enabled = false +filter = exim +action = iptables-multiport[name=exim,port="25,465,587"] +logpath = /var/log/exim/mainlog + +[exim-spam] +enabled = false +filter = exim-spam +action = iptables-multiport[name=exim-spam,port="25,465,587"] +logpath = /var/log/exim/mainlog diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 95ca621f..0d5ae51d 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -8,13 +8,7 @@ # 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html # 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory -# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html -2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 -# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 -2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org 2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted -2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected -2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL). 2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n" 2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n" 2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET") @@ -22,5 +16,4 @@ 2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for : all relevant MX records point to non-existent hosts 2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F= rejected RCPT : Sender verify failed 2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F= rejected RCPT : Unknown user -2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 diff --git a/testcases/files/logs/exim-spam b/testcases/files/logs/exim-spam new file mode 100644 index 00000000..535309f5 --- /dev/null +++ b/testcases/files/logs/exim-spam @@ -0,0 +1,12 @@ +# http://forum.lissyara.su/viewtopic.php?f=20&t=29857 +# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html +# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html +2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 +# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 +2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org +2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected +2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL). +2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 + From 256f60adae92933b2fa7e9099103b95a54b90478 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 3 Jul 2013 09:01:24 +1000 Subject: [PATCH 32/93] DOC: improved log message --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 51835de4..5331e4a0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,7 +16,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released * filter.d/apache-common.conf -- support apache 2.4 more detailed error log format. Closes gh-268 Daniel Black - * action.d/hostsdeny -- ensure permissions are fixed - gh-266 + * action.d/hostsdeny -- Switched to use 'ed' available across all + platforms to ensure permissions are the same before and after a ban - gh-266 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added From 2ffc14359722acff7ddb19f4f1758dab69fb6f8e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 19:49:41 -0400 Subject: [PATCH 33/93] ENH: more of heavydebug'ing for FilterPoll --- server/filterpoll.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/filterpoll.py b/server/filterpoll.py index 20cde0ca..4acecd42 100644 --- a/server/filterpoll.py +++ b/server/filterpoll.py @@ -30,6 +30,11 @@ from mytime import MyTime import time, logging, os +def _ctime(t): + """Given time in seconds provide string representation with milliseconds + """ + return "%s%.3f" %(time.strftime("%X", time.localtime(t)), (t-int(t))) + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -84,6 +89,9 @@ class FilterPoll(FileFilter): def run(self): self.setActive(True) while self._isActive(): + if logSys.getEffectiveLevel() <= 6: + logSys.log(6, "Woke up idle=%s with %d files monitored", + self.getIdle(), len(self.getLogPath())) if not self.getIdle(): # Get file modification for container in self.getLogPath(): @@ -119,6 +127,12 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) self.__file404Cnt[filename] = 0 + if logSys.getEffectiveLevel() <= 7: + # we do not want to waste time on strftime etc if not necessary + dt = logStats.st_mtime - self.__lastModTime[filename] + logSys.log(7, "Checking %s for being modified. Previous/current mtimes: %s / %s. dt: %s", + filename, _ctime(self.__lastModTime[filename]), _ctime(logStats.st_mtime), dt) + # os.system("stat %s | grep Modify" % filename) if self.__lastModTime[filename] == logStats.st_mtime: return False else: From 8c125b6053d137132bacfa7c25cb84e261cd9894 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 19:50:22 -0400 Subject: [PATCH 34/93] ENH: do not sleep 1 sec only on older Pythons while testing filters --- testcases/filtertestcase.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 2381c942..f2f6eb50 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -109,10 +109,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line Returns open fout """ - if sys.version_info[:2] <= (2,4): # pragma: no cover - # on old Python st_mtime is int, so we should give at least 1 sec so - # polling filter could detect the change - time.sleep(1) + # on older Pythons and on some FSs st_mtime is int, so we should + # give it some time so polling filter could detect the change + mtimesleep() if isinstance(fin, str): # pragma: no branch - only used with str in test cases fin = open(fin, 'r') # Skip From f340e5c0f5a74a6bb0b515d9e25dec57f28467c6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 20:43:51 -0400 Subject: [PATCH 35/93] ENH(+possibly BF): for FilterPoll rely not only on mtime but also ino and size to assess if file was modified mtime alone is a poor measure here as many of our tests shown -- on older Pythons and some file systems mtime might be reported only with precision up to a second. If file gets rotated fast, or there are new modifications within the same second, fail2ban might miss them with Polling backend if judging only by mtime. With this modification we will track also inode and size which are all indicative of a file change. --- server/filterpoll.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/filterpoll.py b/server/filterpoll.py index 4acecd42..4cd7cb48 100644 --- a/server/filterpoll.py +++ b/server/filterpoll.py @@ -57,7 +57,7 @@ class FilterPoll(FileFilter): FileFilter.__init__(self, jail) self.__modified = False ## The time of the last modification of the file. - self.__lastModTime = dict() + self.__prevStats = dict() self.__file404Cnt = dict() logSys.debug("Created FilterPoll") @@ -67,7 +67,7 @@ class FilterPoll(FileFilter): # @param path log file path def _addLogPath(self, path): - self.__lastModTime[path] = 0 + self.__prevStats[path] = (0, None, None) # mtime, ino, size self.__file404Cnt[path] = 0 ## @@ -76,7 +76,7 @@ class FilterPoll(FileFilter): # @param path the log file to delete def _delLogPath(self, path): - del self.__lastModTime[path] + del self.__prevStats[path] del self.__file404Cnt[path] ## @@ -126,18 +126,20 @@ class FilterPoll(FileFilter): def isModified(self, filename): try: logStats = os.stat(filename) + stats = logStats.st_mtime, logStats.st_ino, logStats.st_size + pstats = self.__prevStats[filename] self.__file404Cnt[filename] = 0 if logSys.getEffectiveLevel() <= 7: # we do not want to waste time on strftime etc if not necessary - dt = logStats.st_mtime - self.__lastModTime[filename] - logSys.log(7, "Checking %s for being modified. Previous/current mtimes: %s / %s. dt: %s", - filename, _ctime(self.__lastModTime[filename]), _ctime(logStats.st_mtime), dt) + dt = logStats.st_mtime - pstats[0] + logSys.log(7, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s", + filename, pstats, stats, dt) # os.system("stat %s | grep Modify" % filename) - if self.__lastModTime[filename] == logStats.st_mtime: + if pstats == stats: return False else: - logSys.debug(filename + " has been modified") - self.__lastModTime[filename] = logStats.st_mtime + logSys.debug("%s has been modified", filename) + self.__prevStats[filename] = stats return True except OSError, e: logSys.error("Unable to get stat on %s because of: %s" From 052e7ff9da53865a38c28631a6857112ad21bbad Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 20:44:28 -0400 Subject: [PATCH 36/93] ENH: deprecate sophisticated MTimeSleep in favor of no sleeping at all all invocations of mtimesleep() are left in the tests for now --- testcases/utils.py | 57 ++++------------------------------------------ 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/testcases/utils.py b/testcases/utils.py index 6107516b..643c9ad1 100644 --- a/testcases/utils.py +++ b/testcases/utils.py @@ -101,56 +101,7 @@ class FormatterWithTraceBack(logging.Formatter): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) - -class MTimeSleep(object): - """Sleep minimal duration needed to resolve changes in mtime of files in TMPDIR - - mtime resolution depends on Python version AND underlying filesystem - """ - def __init__(self): - self._sleep = None - - @staticmethod - def _get_good_sleep(): - logSys = logging.getLogger("fail2ban.tests") - times = [1.5, 2., 5., 10.] - # we know that older Pythons simply have no ability to resolve - # at < sec level. - if sys.version_info[:2] > (2, 4): - times = [0.1] + times - ffid, name = tempfile.mkstemp() - tfile = os.fdopen(ffid, 'w') - - for stime in times: - prev_stat, dt = "", 0. - # needs to be done 3 times (not clear why) - for i in xrange(3): - stat2 = os.stat(name) - if prev_stat: - dt = (stat2.st_mtime - prev_stat.st_mtime) - prev_stat = stat2 - tfile.write("LOAD\n") - tfile.flush() - time.sleep(stime) - - # check dt but also verify that we are not getting 'quick' - # stime simply by chance of catching second increment - if dt and \ - not (stime < 1 and int(stat2.st_mtime) == stat2.st_mtime): - break - if not dt: - #from warnings import warn - logSys.warn("Could not deduce appropriate sleep time for tests. " - "Maximal tested one of %f sec will be used." % stime) - else: - logSys.debug("It was needed a sleep of %f to detect dt=%f mtime change" - % (stime, dt)) - os.unlink(name) - return stime - - def __call__(self): - if self._sleep is None: - self._sleep = self._get_good_sleep() - time.sleep(self._sleep) - -mtimesleep = MTimeSleep() +def mtimesleep(): + # no sleep now should be necessary since polling tracks now not only + # mtime but also ino and size + pass From 8d25e83f5d154fd2adcaa3cac5f883a8f87b56ad Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 23:37:23 -0400 Subject: [PATCH 37/93] BF: race condition -- file should not be read unless it is not empty Previous code would store md5sum of an empty line as the one identifying the monitored file. That file then was read and possibly failures were found. Upon next "container.open()", md5 digest of now present first line was compared against previous digest of an empty line, which was different, thus file was assumed to be rotated and all the log lines were read once again. The History ----------- In rare cases various tests failed somewhat consistently. Below you can find one case in test_move_file where such failure really made no sense -- we should have not had 4 failures by that point. Fail2ban 0.8.10.dev test suite. Python 2.4.6 (#2, Sep 25 2009, 22:22:06) [GCC 4.3.4]. Please wait... I: Skipping gamin backend testing. Got exception 'No module named gamin' I: Skipping pyinotify backend testing. Got exception 'No module named pyinotify' D: fn=/home/yoh/test/monitorfailures_FilterPoll9nUKoCfail2ban-0 hashes=d41d8cd98f00b204e9800998ecf8427e/d41d8cd98f00b204e9800998ecf8427e inos=5398862/5398862 pos=0 rotate=False D: Adding a ticket for ('193.168.0.128', 1124013599.0, ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']) D: Adding a ticket for ('193.168.0.128', 1124013599.0, ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']) D: Closed with pos 1231 D: fn=/home/yoh/test/monitorfailures_FilterPoll9nUKoCfail2ban-0 hashes=d41d8cd98f00b204e9800998ecf8427e/aeb4e73e6922a746d027eb365ece2149 inos=5398862/5398862 pos=1231 rotate=True D: Adding a ticket for ('193.168.0.128', 1124013599.0, ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']) D: Adding a ticket for ('193.168.0.128', 1124013599.0, ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']) D: Closed with pos 1231 F ====================================================================== FAIL: test_move_file (testcases.filtertestcase.MonitorFailures(/home/yoh/test/monitorfailures_FilterPoll9nUKoCfail2ban)) ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/yoh/deb/gits/fail2ban/testcases/filtertestcase.py", line 451, in test_move_file "Queue must be empty but it is not: %s." AssertionError: Queue must be empty but it is not: server.ticket.FailTicket: ip=193.168.0.128 time=1124013599.0 #attempts=4. ---------------------------------------------------------------------- N.B.1 I preserved here and in the code corresponding additional debug print statements, which are commented out by default. sensible md5 digest was generated by using hexdigest() instead of current a bit faster digest(). Running tests with all the debug output simply breaks the race loose and failure doesn't trigger. N.B.2 d41d8cd98f00b204e9800998ecf8427e is an md5sum of an empty string, and aeb4e73e6922a746d027eb365ece2149 of the first line in that file. --- server/filter.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/server/filter.py b/server/filter.py index 59e612ed..b7c2d091 100644 --- a/server/filter.py +++ b/server/filter.py @@ -21,6 +21,8 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" +import sys + from failmanager import FailManagerEmpty from failmanager import FailManager from ticket import FailTicket @@ -322,6 +324,7 @@ class Filter(JailThread): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) + ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) self.failManager.addFailure(FailTicket(ip, unixTime, [line])) ## @@ -477,7 +480,7 @@ class FileFilter(Filter): return False # Try to open log file. try: - container.open() + has_content = container.open() # see http://python.org/dev/peps/pep-3151/ except IOError, e: logSys.error("Unable to open %s" % filename) @@ -492,7 +495,12 @@ class FileFilter(Filter): logSys.exception(e) return False - while True: + # yoh: has_content is just a bool, so do not expect it to + # change -- loop is exited upon break, and is not entered at + # all if upon container opening that one was empty. If we + # start reading tested to be empty container -- race condition + # might occur leading at least to tests failures. + while has_content: line = container.readline() if (line == "") or not self._isActive(): # The jail reached the bottom or has been stopped @@ -555,10 +563,20 @@ class FileContainer: fd = self.__handler.fileno() flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + # Stat the file before even attempting to read it + stats = os.fstat(self.__handler.fileno()) + if not stats.st_size: + # yoh: so it is still an empty file -- nothing should be + # read from it yet + # print "D: no content -- return" + return False firstLine = self.__handler.readline() # Computes the MD5 of the first line. myHash = md5sum(firstLine).digest() - stats = os.fstat(self.__handler.fileno()) + ## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % ( + ## self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos, + ## self.__hash != myHash or self.__ino != stats.st_ino) + ## sys.stdout.flush() # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: logSys.debug("Log rotation detected for %s" % self.__filename) @@ -567,6 +585,7 @@ class FileContainer: self.__pos = 0 # Sets the file pointer to the last position. self.__handler.seek(self.__pos) + return True def readline(self): if self.__handler is None: @@ -580,6 +599,8 @@ class FileContainer: # Closes the file. self.__handler.close() self.__handler = None + ## print "D: Closed %s with pos %d" % (handler, self.__pos) + ## sys.stdout.flush() From 47ac39fb3451c1d29141aefb452124c05653ef87 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 23:37:41 -0400 Subject: [PATCH 38/93] TST: minor enhancement to test failure msg --- testcases/filtertestcase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index f2f6eb50..d918bbf7 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -447,7 +447,9 @@ def get_monitor_failures_testcase(Filter_): self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time - self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2)) + self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) From 5d7ab9e7fb99287dae3d050ba04a7f5f046f4598 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 23:38:27 -0400 Subject: [PATCH 39/93] DOC: Changelog for preceding changes --- ChangeLog | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 39c0a5aa..dc08641d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,7 +7,7 @@ Fail2Ban (version 0.8.10) 2013/06/12 ================================================================================ -ver. 0.8.11 (2013/XX/XXX) - wanna-be-released +ver. 0.8.11 (2013/XX/XXX) - loves-unittests ----------- - Fixes: @@ -15,6 +15,15 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 * filter.d/apache-common.conf -- support apache 2.4 more detailed error log format. Closes gh-268 + * Backends changes detection and parsing. Close gh-223 and gh-103: + - Polling backend: detect changes in the files not only based on + mtime, but also on the size and inode. It should allow for + better detection of changes and log rotations on busy servers, + older python 2.4, and file systems with precision of mtime only + up to a second (e.g. ext3). + - All backends, possible race condition: do not read from a file + initially reported empty. Originally could have lead to + accounting for detected log lines multiple times. Daniel Black & Мернов Георгий * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in , - New Features: From 5df6796e6943807e17b0e5ea962613af8f7b189f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 23:51:06 -0400 Subject: [PATCH 40/93] ENH: DNS resolution -- catch parent exception IMHO there is no good reason to capture only gaierror. on my network it was consistent to error out with ====================================================================== ERROR: testIgnoreIPNOK (testcases.filtertestcase.IgnoreIP) ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/yoh/deb/gits/fail2ban/testcases/filtertestcase.py", line 166, in testIgnoreIPNOK self.assertFalse(self.filter.inIgnoreIPList(ip)) File "/home/yoh/deb/gits/fail2ban/server/filter.py", line 277, in inIgnoreIPList ips = DNSUtils.dnsToIp(i) File "/home/yoh/deb/gits/fail2ban/server/filter.py", line 625, in dnsToIp return socket.gethostbyname_ex(dns)[2] error: [Errno 11] Resource temporarily unavailable with this commit tests would pass normally as they should --- server/filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/filter.py b/server/filter.py index b7c2d091..6cd37ca1 100644 --- a/server/filter.py +++ b/server/filter.py @@ -623,9 +623,9 @@ class DNSUtils: """ try: return socket.gethostbyname_ex(dns)[2] - except socket.gaierror: - logSys.warn("Unable to find a corresponding IP address for %s" - % dns) + except socket.error, e: + logSys.warn("Unable to find a corresponding IP address for %s: %s" + % (dns, e)) return list() dnsToIp = staticmethod(dnsToIp) From 404574499d9d76c04e07c36b09a1fbcfb1361795 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 2 Jul 2013 23:52:37 -0400 Subject: [PATCH 41/93] BF: fail2ban-testcases -- use full "warning" instead of warn for the verbosity dictionary --- fail2ban-testcases | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index fc5cc146..89fc7deb 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -79,7 +79,7 @@ logSys = logging.getLogger("fail2ban") verbosity = {'heavydebug': 4, 'debug': 3, 'info': 2, - 'warn': 1, + 'warning': 1, 'error': 1, 'fatal': 0, None: 1}[opts.log_level] From e282d6b1c7aa15281be3fb748c11ad70ca5dd035 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 3 Jul 2013 00:09:39 -0400 Subject: [PATCH 42/93] ENH: Remove unused any longer _ctime helper --- server/filterpoll.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/filterpoll.py b/server/filterpoll.py index 4cd7cb48..8b4038b8 100644 --- a/server/filterpoll.py +++ b/server/filterpoll.py @@ -30,11 +30,6 @@ from mytime import MyTime import time, logging, os -def _ctime(t): - """Given time in seconds provide string representation with milliseconds - """ - return "%s%.3f" %(time.strftime("%X", time.localtime(t)), (t-int(t))) - # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") From 2155f6bfa58ed820d95ce50ce7488d6c43c31a20 Mon Sep 17 00:00:00 2001 From: Alexander Dietrich Date: Thu, 4 Jul 2013 08:57:52 +0200 Subject: [PATCH 43/93] Update ChangeLog and jail.conf example --- ChangeLog | 3 +++ config/jail.conf | 2 ++ 2 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 39c0a5aa..dd5426a2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -35,6 +35,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released * fail2ban-client -- log to standard error. Closes gh-264 * Fail to configure if not a single log file was found for an enabled jail. Closes gh-63 + Alexander Dietrich + * action.d/sendmail-common.conf -- added common sendmail settings file + and made the sender display name configurable ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/config/jail.conf b/config/jail.conf index c999cc7b..e4e7e5b0 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -66,6 +66,8 @@ enabled = false filter = sshd action = iptables[name=SSH, port=ssh, protocol=tcp] sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com] +# Alternative example: +# sendmail-whois[name=SSH, sendername="Fail2Ban - example.com"] logpath = /var/log/sshd.log maxretry = 5 From 04b8069ceed9212ef9b72ec2ea3d8b8030a31685 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 5 Jul 2013 10:12:29 -0400 Subject: [PATCH 44/93] ENH: adjust sendmail-whois 'active' example to have also sendername in it --- config/jail.conf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index e4e7e5b0..740db30c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -65,9 +65,7 @@ usedns = warn enabled = false filter = sshd action = iptables[name=SSH, port=ssh, protocol=tcp] - sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com] -# Alternative example: -# sendmail-whois[name=SSH, sendername="Fail2Ban - example.com"] + sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] logpath = /var/log/sshd.log maxretry = 5 From bfa2b9dec3bb5414e8a8f46d518ec3bce2fbec1a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 5 Jul 2013 18:36:02 +0100 Subject: [PATCH 45/93] ENH: dovecot filter additions for session, time value and blank user --- ChangeLog | 3 +++ config/filter.d/dovecot.conf | 2 +- testcases/files/logs/dovecot | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index dd5426a2..c91d6bcd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released Alexander Dietrich * action.d/sendmail-common.conf -- added common sendmail settings file and made the sender display name configurable + Steven Hiscocks + * filter.d/dovecot - Addition of session, time values and possible blank + user ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index f111859f..2143e224 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -17,7 +17,7 @@ _daemon = dovecot(-auth)? # Values: TEXT # failregex = ^%(__prefix_line)s(pam_unix(\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use disabled \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 01df0af3..b4d978b6 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -14,3 +14,5 @@ Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 +Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= +Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= From 619603fe054765da7bcfed374c55ce07c142abeb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 7 Jul 2013 17:48:20 +1000 Subject: [PATCH 46/93] BF: match asterisk InvalidPassword correctly --- config/filter.d/asterisk.conf | 2 +- testcases/files/logs/asterisk | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 5e775430..e4e5de5f 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -32,7 +32,7 @@ failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' ^%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ ^%(log_prefix)s Failed to authenticate user [^@]+@\S*$ ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S*$ - ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ + ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"(,Challenge="\w+",ReceivedChallenge="\w+")?(,ReceivedHash="[\da-f]+")?$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index e2fe3b6e..b4ecf974 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -19,3 +19,4 @@ [2009-12-22 16:35:24] NOTICE[1570][C-00000086] chan_sip.c: Sending fake auth rejection for device 1022;tag=5d8b6f92 # http://www.spinics.net/lists/asterisk/msg127381.html [2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 +[2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39" From 1f5097649c55925e7b51c3156d5e0fdd0627a21e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 9 Jul 2013 08:20:13 +1000 Subject: [PATCH 47/93] DOC: ChangeLog for exim-spam.conf filter and tabs->spaces in changelog --- ChangeLog | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 549eb7f7..4a28f815 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,11 +13,14 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released - Fixes: Yaroslav Halchenko * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 - Daniel Black & Мернов Георгий - * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in , + Daniel Black & Мернов Георгий + * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in , - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added + Daniel Black + * filter.d/exim-spam.conf -- a splitout of exim's spam regexes + with additions for greater control over filtering spam. - Enhancements: Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening From 174e3dba6dc5bfc44d34f02c68f5dee459cc7b92 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 9 Jul 2013 08:36:53 +1000 Subject: [PATCH 48/93] DOC: Note on new dependency - ed for hosts_deny --- ChangeLog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5331e4a0..d02601c1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,8 +16,9 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released * filter.d/apache-common.conf -- support apache 2.4 more detailed error log format. Closes gh-268 Daniel Black - * action.d/hostsdeny -- Switched to use 'ed' available across all - platforms to ensure permissions are the same before and after a ban - gh-266 + * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across + all platforms to ensure permissions are the same before and after a ban - + closes gh-266 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added From 606e97683baf3216be2c521305b21122f500a586 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 12 Jul 2013 23:34:04 +0100 Subject: [PATCH 49/93] BF: jail.conf multiport actions previously using single port iptables --- config/jail.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c41d2c1b..e3b92038 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -229,7 +229,7 @@ logpath = /var/log/apache2/error_log enabled = false filter = roundcube-auth -action = iptables[name=RoundCube, port="http,https"] +action = iptables-multiport[name=RoundCube, port="http,https"] logpath = /var/log/roundcube/userlogins @@ -241,7 +241,7 @@ enabled = false filter = sogo-auth # without proxy this would be: # port = 20000 -action = iptables[name=SOGo, port="http,https"] +action = iptables-multiport[name=SOGo, port="http,https"] logpath = /var/log/sogo/sogo.log # Ban attackers that try to use PHP's URL-fopen() functionality @@ -251,7 +251,7 @@ logpath = /var/log/sogo/sogo.log [php-url-fopen] enabled = false -action = iptables[name=php-url-open, port="http,https"] +action = iptables-multiport[name=php-url-open, port="http,https"] filter = php-url-fopen logpath = /var/www/*/logs/access_log maxretry = 1 @@ -267,7 +267,7 @@ maxretry = 1 enabled = false filter = lighttpd-fastcgi -action = iptables[name=lighttpd-fastcgi, port="http,https"] +action = iptables-multiport[name=lighttpd-fastcgi, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 @@ -279,7 +279,7 @@ maxretry = 2 enabled = false filter = lighttpd-auth -action = iptables[name=lighttpd-auth, port="http,https"] +action = iptables-multiport[name=lighttpd-auth, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 @@ -405,7 +405,7 @@ maxretry=5 enabled = false filter = 3proxy -action = iptables-multiport[name=3proxy, port=3128, protocol=tcp] +action = iptables[name=3proxy, port=3128, protocol=tcp] logpath = /var/log/3proxy.log [exim] From 40f67c64b8d8d25a6d887ca83c9527549113ef41 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 13 Jul 2013 23:03:01 +0100 Subject: [PATCH 50/93] TST: Test sample logs' entries are matched by filter regexs --- fail2ban-testcases | 3 + server/filter.py | 9 +- testcases/files/logs/3proxy | 3 + testcases/files/logs/apache-auth | 2 + testcases/files/logs/apache-nohome | 2 + testcases/files/logs/apache-noscript | 1 + testcases/files/logs/apache-overflows | 2 + testcases/files/logs/assp | 12 +++ testcases/files/logs/asterisk | 18 ++++ testcases/files/logs/dovecot | 17 +++- testcases/files/logs/exim | 12 +++ testcases/files/logs/exim-spam | 5 + .../files/logs/{lighttpd => lighttpd-auth} | 2 + .../files/logs/{mysqld.log => mysqld-auth} | 6 ++ testcases/files/logs/named-refused | 6 ++ testcases/files/logs/pam-generic | 7 ++ testcases/files/logs/postfix | 1 + testcases/files/logs/proftpd | 6 ++ testcases/files/logs/pure-ftpd | 2 + testcases/files/logs/roundcube-auth | 2 + testcases/files/logs/sasl | 2 + testcases/files/logs/sogo-auth | 14 +++ testcases/files/logs/sshd | 28 ++++++ testcases/files/logs/sshd-ddos | 1 + testcases/files/logs/vsftpd | 3 + testcases/files/logs/webmin-auth | 2 + testcases/files/logs/{wu-ftpd => wuftpd} | 1 + testcases/samplestestcase.py | 98 +++++++++++++++++++ 28 files changed, 261 insertions(+), 6 deletions(-) rename testcases/files/logs/{lighttpd => lighttpd-auth} (60%) rename testcases/files/logs/{mysqld.log => mysqld-auth} (51%) rename testcases/files/logs/{wu-ftpd => wuftpd} (60%) create mode 100644 testcases/samplestestcase.py diff --git a/fail2ban-testcases b/fail2ban-testcases index 89fc7deb..278dd890 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -32,6 +32,7 @@ from testcases import banmanagertestcase from testcases import clientreadertestcase from testcases import failmanagertestcase from testcases import filtertestcase +from testcases import samplestestcase from testcases import servertestcase from testcases import datedetectortestcase from testcases import actiontestcase @@ -170,6 +171,8 @@ tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) +# Filter Regex tests with sample logs +tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) # # Extensive use-tests of different available filters backends diff --git a/server/filter.py b/server/filter.py index 6cd37ca1..d0390e62 100644 --- a/server/filter.py +++ b/server/filter.py @@ -284,7 +284,7 @@ class Filter(JailThread): return False - def processLine(self, line): + def processLine(self, line, returnRawHost=False): """Split the time portion from log msg and return findFailures on them """ try: @@ -306,7 +306,7 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine) + return self.findFailure(timeLine, logLine, returnRawHost) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -348,7 +348,7 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, timeLine, logLine): + def findFailure(self, timeLine, logLine, returnRawHost=False): failList = list() # Checks if we must ignore this line. if self.ignoreLine(logLine): @@ -371,6 +371,9 @@ class Filter(JailThread): else: try: host = failRegex.getHost() + if returnRawHost: + failList.append([host, date]) + break ipMatch = DNSUtils.textToIp(host, self.__useDns) if ipMatch: for ip in ipMatch: diff --git a/testcases/files/logs/3proxy b/testcases/files/logs/3proxy index 2967c9bf..614600ef 100644 --- a/testcases/files/logs/3proxy +++ b/testcases/files/logs/3proxy @@ -1,3 +1,6 @@ +# failJSON: { "time": "2013-06-11T02:09:40", "match": true , "host": "1.2.3.4" } 11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +# failJSON: { "time": "2013-06-11T02:09:43", "match": true , "host": "1.2.3.4" } 11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +# failJSON: { "time": "2013-06-13T01:39:34", "match": true , "host": "1.2.3.4" } 13-06-2013 01:39:34 +0300 PROXY.3128 00508 - 1.2.3.4:28938 0.0.0.0:0 0 0 0 diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index cf0f6d30..6035db3f 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -1,5 +1,7 @@ # Should not match -- DoS vector https://vndh.net/note:fail2ban-089-denial-service +# failJSON: { "match": false } [Sat Jun 01 02:17:42 2013] [error] [client 192.168.33.1] File does not exist: /srv/http/site/[client 192.168.0.1] user root not found # should match +# failJSON: { "time": "2005-06-01T02:17:42", "match": true , "host": "192.168.0.2" } [Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found diff --git a/testcases/files/logs/apache-nohome b/testcases/files/logs/apache-nohome index e8ef56bc..aea0d816 100644 --- a/testcases/files/logs/apache-nohome +++ b/testcases/files/logs/apache-nohome @@ -1,4 +1,6 @@ # Apache 2.2 +# failJSON: { "time": "2005-06-01T11:23:08", "match": true , "host": "1.2.3.4" } [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] File does not exist: /xxx/~ # Apache 2.4 +# failJSON: { "time": "2005-06-27T11:55:44", "match": true , "host": "192.0.2.12" } [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 192.0.2.12:46652] AH00128: File does not exist: /xxx/~ diff --git a/testcases/files/logs/apache-noscript b/testcases/files/logs/apache-noscript index 5d5d35ff..e08b3468 100644 --- a/testcases/files/logs/apache-noscript +++ b/testcases/files/logs/apache-noscript @@ -1 +1,2 @@ +# failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" } [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 18a44bfc..1af377f1 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,2 +1,4 @@ +# failJSON: { "time": "2005-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 +# failJSON: { "time": "2005-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 diff --git a/testcases/files/logs/assp b/testcases/files/logs/assp index 99363001..2c658eb9 100644 --- a/testcases/files/logs/assp +++ b/testcases/files/logs/assp @@ -1,13 +1,25 @@ +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:10:37", "match": true , "host": "68.171.223.68" } Apr-07-13 07:10:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:12:37", "match": true , "host": "68.171.223.68" } Apr-07-13 07:12:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:14:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:14:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (8); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (9); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (10); +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index b4ecf974..ec2fec7c 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -1,22 +1,40 @@ # Sample log files for asterisk +# failJSON: { "time": "2012-02-13T17:21:54", "match": true , "host": "1.2.3.4" } [2012-02-13 17:21:54] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Wrong password +# failJSON: { "time": "2012-02-13T17:18:22", "match": true , "host": "1.2.3.4" } [2012-02-13 17:18:22] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - No matching peer found +# failJSON: { "time": "2012-02-13T17:21:21", "match": true , "host": "1.2.3.4" } [2012-02-13 17:21:21] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Username/auth name mismatch +# failJSON: { "time": "2012-02-13T17:32:01", "match": true , "host": "1.2.3.4" } [2012-02-13 17:32:01] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Device does not match ACL +# failJSON: { "time": "2012-02-13T17:34:10", "match": true , "host": "1.2.3.4" } [2012-02-13 17:34:10] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Peer is not supposed to register +# failJSON: { "time": "2012-02-13T17:36:23", "match": true , "host": "1.2.3.4" } [2012-02-13 17:36:23] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - ACL error (permit/deny) +# failJSON: { "time": "2012-02-13T17:53:59", "match": true , "host": "1.2.3.4" } [2012-02-13 17:53:59] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed to authenticate as 'Fail2ban' +# failJSON: { "time": "2012-02-13T17:39:20", "match": true , "host": "1.2.3.4" } [2012-02-13 17:39:20] NOTICE[1638] chan_iax2.c: No registration for peer 'Fail2ban' (from 1.2.3.4) +# failJSON: { "time": "2012-02-13T17:44:26", "match": true , "host": "1.2.3.4" } [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) +# failJSON: { "time": "2012-02-13T17:37:07", "match": true , "host": "1.2.3.4" } [2012-02-13 17:37:07] NOTICE[1638] chan_sip.c: Failed to authenticate user "Fail2ban" ;tag=1r698745234 +# failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. +# failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found +# failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" } [2013-05-13 07:10:53] SECURITY[1204] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="1368439853-500975",Severity="Error",Service="SIP",EventVersion="1",AccountID="00972599580679",SessionID="0x7f8ecc0421f8",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/1.2.3.4/5070" +# failJSON: { "time": "2013-06-10T18:15:03", "match": true , "host": "1.2.3.4" } [2013-06-10 18:15:03] NOTICE[2723] chan_sip.c: Registration from '"100"' failed for '1.2.3.4' - Not a local domain # http://forum.4psa.com/showthread.php?t=6601 +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } [2009-12-22 16:35:24] NOTICE[6163] chan_sip.c: Sending fake auth rejection for device ;tag=e3793a95e1acbc69o # http://www.freepbx.org/forum/general-help/fake-auth-rejection +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } [2009-12-22 16:35:24] NOTICE[1570][C-00000086] chan_sip.c: Sending fake auth rejection for device 1022;tag=5d8b6f92 # http://www.spinics.net/lists/asterisk/msg127381.html +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } [2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 +# failJSON: { "time": "2013-07-06T09:09:25", "match": true , "host": "141.255.164.106" } [2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39" diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 01df0af3..19297912 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1,16 +1,27 @@ +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" } @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root +# failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 +# failJSON: { "time": "2005-06-13T16:30:54", "match": true , "host": "49.176.98.87" } Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected +# failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index 0d5ae51d..95c70d25 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -1,19 +1,31 @@ # From IRC 2013-01-04 +# failJSON: { "time": "2013-01-04T17:03:46", "match": true , "host": "24.106.174.74" } 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) # From IRC 2013-06-13 XATRIX (Georgiy Mernov) +# failJSON: { "time": "2013-06-12T03:57:58", "match": true , "host": "120.196.140.45" } 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) +# failJSON: { "time": "2013-06-12T13:18:11", "match": true , "host": "101.66.165.86" } 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data +# failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "193.169.56.211" } 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address # http://forum.lissyara.su/viewtopic.php?f=20&t=29857 # 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html # 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# failJSON: { "time": "2013-06-10T18:33:32", "match": true , "host": "202.132.70.178" } 2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted +# failJSON: { "time": "2013-06-02T06:54:20", "match": true , "host": "211.148.195.192" } 2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n" +# failJSON: { "time": "2013-06-02T09:05:48", "match": true , "host": "82.96.160.77" } 2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n" +# failJSON: { "time": "2013-06-02T09:22:05", "match": true , "host": "163.14.21.161" } 2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET") +# failJSON: { "time": "2013-06-02T15:06:18", "match": true , "host": "46.20.35.114" } 2013-06-02 15:06:18 H=(VM-WIN2K3-1562) [46.20.35.114] sender verify fail for : Unknown user +# failJSON: { "time": "2013-06-07T02:02:09", "match": true , "host": "91.232.21.92" } 2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for : all relevant MX records point to non-existent hosts +# failJSON: { "time": "2013-06-15T16:34:55", "match": true , "host": "182.18.24.93" } 2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F= rejected RCPT : Sender verify failed +# failJSON: { "time": "2013-06-15T16:36:49", "match": true , "host": "111.67.203.116" } 2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F= rejected RCPT : Unknown user diff --git a/testcases/files/logs/exim-spam b/testcases/files/logs/exim-spam index 535309f5..46b915d1 100644 --- a/testcases/files/logs/exim-spam +++ b/testcases/files/logs/exim-spam @@ -3,10 +3,15 @@ # http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html # 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory # http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html +# failJSON: { "time": "2004-01-18T07:15:35", "match": true , "host": "4.47.28.40" } 2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 # https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 +# failJSON: { "time": "2013-06-15T11:19:33", "match": true , "host": "2.181.148.95" } 2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org +# failJSON: { "time": "2013-06-09T10:21:28", "match": true , "host": "46.254.240.82" } 2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected +# failJSON: { "time": "2013-06-15T11:20:36", "match": true , "host": "83.235.177.148" } 2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL). +# failJSON: { "time": "2013-06-16T02:50:43", "match": true , "host": "111.67.203.114" } 2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 diff --git a/testcases/files/logs/lighttpd b/testcases/files/logs/lighttpd-auth similarity index 60% rename from testcases/files/logs/lighttpd rename to testcases/files/logs/lighttpd-auth index c3cfcb75..745a2462 100644 --- a/testcases/files/logs/lighttpd +++ b/testcases/files/logs/lighttpd-auth @@ -1,3 +1,5 @@ #authentification failure (mod_auth) +# failJSON: { "time": "2011-12-25T17:09:20", "match": true , "host": "4.4.4.4" } 2011-12-25 17:09:20: (http_auth.c.875) password doesn't match for /gitweb/ username: francois, IP: 4.4.4.4 +# failJSON: { "time": "2012-09-26T10:24:35", "match": true , "host": "4.4.4.4" } 2012-09-26 10:24:35: (http_auth.c.1136) digest: auth failed for xxx : wrong password, IP: 4.4.4.4 diff --git a/testcases/files/logs/mysqld.log b/testcases/files/logs/mysqld-auth similarity index 51% rename from testcases/files/logs/mysqld.log rename to testcases/files/logs/mysqld-auth index b3a73078..757ecb66 100644 --- a/testcases/files/logs/mysqld.log +++ b/testcases/files/logs/mysqld-auth @@ -1,6 +1,12 @@ +# failJSON: { "time": "2013-03-24T00:04:00", "match": true , "host": "192.168.1.35" } 130324 0:04:00 [Warning] Access denied for user 'root'@'192.168.1.35' (using password: NO) +# failJSON: { "time": "2013-03-24T08:24:09", "match": true , "host": "220.95.238.171" } 130324 8:24:09 [Warning] Access denied for user 'root'@'220.95.238.171' (using password: NO) +# failJSON: { "time": "2013-03-24T17:56:13", "match": true , "host": "61.160.223.112" } 130324 17:56:13 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: NO) +# failJSON: { "time": "2013-03-24T17:56:14", "match": true , "host": "61.160.223.112" } 130324 17:56:14 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: YES) +# failJSON: { "time": "2013-03-24T19:01:39", "match": true , "host": "61.147.108.35" } 130324 19:01:39 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: NO) +# failJSON: { "time": "2013-03-24T19:01:40", "match": true , "host": "61.147.108.35" } 130324 19:01:40 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: YES) diff --git a/testcases/files/logs/named-refused b/testcases/files/logs/named-refused index 130e7417..6f6092e2 100644 --- a/testcases/files/logs/named-refused +++ b/testcases/files/logs/named-refused @@ -1,6 +1,12 @@ +# failJSON: { "time": "2005-07-24T14:16:55", "match": true , "host": "194.145.196.18" } Jul 24 14:16:55 raid5 named[3935]: client 194.145.196.18#4795: query 'ricreig.com/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:16:56", "match": true , "host": "62.123.164.113" } Jul 24 14:16:56 raid5 named[3935]: client 62.123.164.113#32768: query 'ricreig.com/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:17:13", "match": true , "host": "148.160.29.6" } Jul 24 14:17:13 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'geo-mueller.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:20:25", "match": true , "host": "148.160.29.6" } Jul 24 14:20:25 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'shivaree.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "148.160.29.6" } Jul 24 14:23:36 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'mietberatung.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "62.109.4.89" } Jul 24 14:23:36 raid5 named[3935]: client 62.109.4.89#9334: view external: query (cache) './NS/IN' denied diff --git a/testcases/files/logs/pam-generic b/testcases/files/logs/pam-generic index d84ab153..dc7efeee 100644 --- a/testcases/files/logs/pam-generic +++ b/testcases/files/logs/pam-generic @@ -1,7 +1,14 @@ +# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1" } Feb 7 15:10:42 example pure-ftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=pure-ftpd ruser=sample-user rhost=192.168.1.1 +# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } May 12 09:47:54 vaio sshd[16004]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com user=root +# failJSON: { "time": "2005-05-12T09:48:03", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } May 12 09:48:03 vaio sshd[16021]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com +# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62" } May 15 18:02:12 localhost proftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=66.232.129.62 user=mark +# failJSON: { "time": "2004-11-25T17:12:13", "match": true , "host": "192.168.10.3" } Nov 25 17:12:13 webmail pop(pam_unix)[4920]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.168.10.3 user=mailuser +# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" } Jul 19 18:11:26 srv2 vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com +# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" } Jul 19 18:11:26 srv2 vsftpd: pam_unix: authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 28ff95a4..50e3a360 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -1,3 +1,4 @@ # per https://github.com/fail2ban/fail2ban/issues/125 # and https://github.com/fail2ban/fail2ban/issues/126 +# failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" } Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index aaaf0295..404ebfaf 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -1,8 +1,14 @@ +# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" } Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers +# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" } Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 +# failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" } Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. +# failJSON: { "time": "2005-06-09T11:15:43", "match": true , "host": "101.71.143.238" } Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21 +# failJSON: { "time": "2005-06-13T22:07:23", "match": true , "host": "59.167.242.100" } Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. +# failJSON: { "time": "2005-06-14T00:09:59", "match": true , "host": "59.167.242.100" } Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 diff --git a/testcases/files/logs/pure-ftpd b/testcases/files/logs/pure-ftpd index 4b4e3455..ef49b2ff 100644 --- a/testcases/files/logs/pure-ftpd +++ b/testcases/files/logs/pure-ftpd @@ -1,2 +1,4 @@ +# failJSON: { "time": "2005-01-31T16:54:07", "match": true , "host": "24.79.92.194" } Jan 31 16:54:07 desktop pure-ftpd: (?@24.79.92.194) [WARNING] Authentication failed for user [Administrator] +# failJSON: { "time": "2004-11-05T18:54:02", "match": true , "host": "server202181210195.ixlink.net" } Nov 5 18:54:02 pure-ftpd: (?@server202181210195.ixlink.net) [WARNING] Authentication failed for user [Administrator] diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index 04e0faf5..dec8051e 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -1,2 +1,4 @@ +# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" } [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 +# failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" } May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 diff --git a/testcases/files/logs/sasl b/testcases/files/logs/sasl index 18c5ff14..ca306098 100644 --- a/testcases/files/logs/sasl +++ b/testcases/files/logs/sasl @@ -1,5 +1,7 @@ #1 Example from postfix from dbts #507990 +# failJSON: { "time": "2004-12-02T22:24:22", "match": true , "host": "114.44.142.233" } Dec 2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.net[114.44.142.233]: SASL CRAM-MD5 authentication failed: PDc3OTEwNTkyNTEyMzA2NDIuMTIyODI1MzA2MUBoZWw+ #2 Example from postfix from dbts #573314 +# failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" } Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure diff --git a/testcases/files/logs/sogo-auth b/testcases/files/logs/sogo-auth index e4fb9409..02a69c6d 100644 --- a/testcases/files/logs/sogo-auth +++ b/testcases/files/logs/sogo-auth @@ -1,17 +1,31 @@ # yoh: Kept original apache log lines as well, just in case they might come useful # for (testing) multiline regular expressions which would further constraint # SOGo log lines +# failJSON: { "match": false } Mar 24 08:58:32 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=hack0r,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:32", "match": true , "host": "173.194.44.31" } Mar 24 08:58:32 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'hack0r' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:32 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/38 0.311 - - 2M +# failJSON: { "match": false } Mar 24 08:58:40 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=kiddy,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:40", "match": true , "host": "173.194.44.31" } Mar 24 08:58:40 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'kiddy' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:40 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/37 0.007 - - 32K +# failJSON: { "match": false } Mar 24 08:58:50 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=plsBanMe,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:50", "match": true , "host": "173.194.44.31" } Mar 24 08:58:50 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'plsBanMe' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:50 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/40 0.008 - - 0 +# failJSON: { "match": false } Mar 24 08:58:59 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=root,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:59", "match": true , "host": "173.194.44.31" } Mar 24 08:58:59 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'root' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:59 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/36 0.007 - - 0 +# failJSON: { "match": false } Mar 24 08:59:04 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=admin,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:59:04", "match": true , "host": "173.194.44.31" } Mar 24 08:59:04 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'admin' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 1943fb37..ff97c5a5 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -1,51 +1,79 @@ #1 +# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.030.0.6" } Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6 +# failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" } May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com #2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345 #3 +# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 +# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 #4 +# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "211.114.51.213" } Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 211.114.51.213 #5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch # yoh: added ':' after [sshd] since the case without is not really common any more +# failJSON: { "time": "2005-03-03T00:17:22", "match": true , "host": "211.188.220.49" } Mar 3 00:17:22 [sshd]: User root from 211.188.220.49 not allowed because not listed in AllowUsers +# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "example.com" } Feb 25 14:34:11 belka sshd[31607]: User root from example.com not allowed because not listed in AllowUsers #6 ew filter introduced thanks to report Guido Bozzetto +# failJSON: { "time": "2004-11-11T23:33:27", "match": true , "host": "218.249.210.161" } Nov 11 23:33:27 Server sshd[5174]: refused connect from _U2FsdGVkX19P3BCJmFBHhjLza8BcMH06WCUVwttMHpE=_@::ffff:218.249.210.161 (::ffff:218.249.210.161) #7 added exclamation mark to BREAK-IN # Now should be a negative since we decided not to catch those +# failJSON: { "match": false } Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT +# failJSON: { "match": false } Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! #8 DenyUsers https://github.com/fail2ban/fail2ban/issues/47 +# failJSON: { "time": "2005-04-16T22:01:15", "match": true , "host": "46.45.128.3" } Apr 16 22:01:15 al-ribat sshd[5154]: User root from 46.45.128.3 not allowed because listed in DenyUsers #9 OpenSolaris patch - pull https://github.com/fail2ban/fail2ban/pull/182 +# failJSON: { "time": "2005-03-29T05:59:23", "match": true , "host": "205.186.180.55" } Mar 29 05:59:23 dusky sshd[20878]: [ID 800047 auth.info] Failed keyboard-interactive for from 205.186.180.55 port 42742 ssh2 +# failJSON: { "time": "2005-03-29T05:20:09", "match": true , "host": "205.186.180.30" } Mar 29 05:20:09 dusky sshd[19558]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.30 port 54520 ssh2 #10 OSX syslog error +# failJSON: { "time": "2005-04-29T17:16:20", "match": true , "host": "example.com" } Apr 29 17:16:20 Jamess-iMac.local sshd[62312]: error: PAM: authentication error for james from example.com via 192.168.1.201 +# failJSON: { "time": "2005-04-29T20:11:08", "match": true , "host": "205.186.180.35" } Apr 29 20:11:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for from 205.186.180.35 port 42742 ssh2 +# failJSON: { "time": "2005-04-29T20:12:08", "match": true , "host": "205.186.180.22" } Apr 29 20:12:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.22 port 54520 ssh2 +# failJSON: { "time": "2005-04-29T20:13:08", "match": true , "host": "205.186.180.42" } Apr 29 20:13:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for james from 205.186.180.42 port 54520 ssh2 +# failJSON: { "time": "2005-04-29T20:14:08", "match": true , "host": "205.186.180.44" } Apr 29 20:14:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for from 205.186.180.44 port 42742 ssh2 +# failJSON: { "time": "2005-04-30T01:42:12", "match": true , "host": "205.186.180.77" } Apr 30 01:42:12 Jamess-iMac.local sshd[2554]: Failed keyboard-interactive/pam for invalid user jamedds from 205.186.180.77 port 33723 ssh2 +# failJSON: { "time": "2005-04-29T12:53:38", "match": true , "host": "205.186.180.88" } Apr 29 12:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication failure for james from 205.186.180.88 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T13:53:38", "match": true , "host": "205.186.180.99" } Apr 29 13:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication failure for james from 205.186.180.99 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T15:53:38", "match": true , "host": "205.186.180.100" } Apr 29 15:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication error for james from 205.186.180.100 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T16:53:38", "match": true , "host": "205.186.180.101" } Apr 29 16:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.101 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T17:53:38", "match": true , "host": "205.186.180.102" } Apr 29 17:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.102 +# failJSON: { "time": "2005-04-29T18:53:38", "match": true , "host": "205.186.180.103" } Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.103 #11 https://github.com/fail2ban/fail2ban/issues/267 There might be no colon after [daemon] +# failJSON: { "time": "2005-06-25T23:53:34", "match": true , "host": "1.2.3.4" } Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers diff --git a/testcases/files/logs/sshd-ddos b/testcases/files/logs/sshd-ddos index d71c6bb2..452abbde 100644 --- a/testcases/files/logs/sshd-ddos +++ b/testcases/files/logs/sshd-ddos @@ -1,2 +1,3 @@ # http://forums.powervps.com/showthread.php?t=1667 +# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" } Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 diff --git a/testcases/files/logs/vsftpd b/testcases/files/logs/vsftpd index a8b6a4cf..f3fb997f 100644 --- a/testcases/files/logs/vsftpd +++ b/testcases/files/logs/vsftpd @@ -1,7 +1,10 @@ #1 PAM based +# failJSON: { "time": "2004-10-11T01:06:47", "match": true , "host": "209.67.1.67" } Oct 11 01:06:47 ServerJV vsftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=209.67.1.67 +# failJSON: { "time": "2005-02-06T12:02:29", "match": true , "host": "64.168.103.1" } Feb 6 12:02:29 server vsftpd(pam_unix)[15522]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=64.168.103.1 user=user1 #2 Internal +# failJSON: { "time": "2005-01-19T12:20:33", "match": true , "host": "64.106.46.98" } Fri Jan 19 12:20:33 2007 [pid 27202] [anonymous] FAIL LOGIN: Client "64.106.46.98" diff --git a/testcases/files/logs/webmin-auth b/testcases/files/logs/webmin-auth index 86d3f321..424c114d 100644 --- a/testcases/files/logs/webmin-auth +++ b/testcases/files/logs/webmin-auth @@ -1,7 +1,9 @@ #Webmin authentication failures from /var/log/auth.log #1 User exists, bad password +# failJSON: { "time": "2004-12-13T08:15:18", "match": true , "host": "89.2.49.230" } Dec 13 08:15:18 sb1 webmin[25875]: Invalid login as root from 89.2.49.230 #2 User does not exists +# failJSON: { "time": "2004-12-12T23:14:19", "match": true , "host": "188.40.105.142" } Dec 12 23:14:19 sb1 webmin[22134]: Non-existent login as robert from 188.40.105.142 diff --git a/testcases/files/logs/wu-ftpd b/testcases/files/logs/wuftpd similarity index 60% rename from testcases/files/logs/wu-ftpd rename to testcases/files/logs/wuftpd index b6b41613..22ac0303 100644 --- a/testcases/files/logs/wu-ftpd +++ b/testcases/files/logs/wuftpd @@ -1,2 +1,3 @@ # This login line is from syslog +# failJSON: { "time": "2004-10-06T09:59:26", "match": true , "host": "202.108.145.173" } Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py new file mode 100644 index 00000000..2bb73ca1 --- /dev/null +++ b/testcases/samplestestcase.py @@ -0,0 +1,98 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__copyright__ = "Copyright (c) 2013 Steven Hiscocks" +__license__ = "GPL" + +import unittest, os, fileinput, re, json, datetime + +from server.filter import Filter +from client.filterreader import FilterReader + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") +CONFIG_DIR = "config" + +class FilterSamplesRegex(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + self.filter = Filter(None) + self.filter.setActive(True) + + def tearDown(self): + """Call after every test case.""" + +def testSampleRegexsFactory(name): + def testFilter(self): + + # Check filter exists + filterConf = FilterReader(name, "jail", basedir=CONFIG_DIR) + filterConf.read() + filterConf.getOptions({}) + + for opt in filterConf.convert(): + if opt[2] == "addfailregex": + self.filter.addFailRegex(opt[3]) + + logFile = fileinput.FileInput( + os.path.join(TEST_FILES_DIR, "logs", name)) + for line in logFile: + jsonREMatch = re.match("^# ?failJSON:(.+)$", line) + if jsonREMatch: + try: + faildata = json.loads(jsonREMatch.group(1)) + except ValueError, e: + raise ValueError("%s: %s:%i" % + (e, logFile.filename(), logFile.filelineno())) + line = next(logFile) + elif line.startswith("#") or not line.strip(): + continue + else: + faildata = {} + + ret = self.filter.processLine(line, returnRawHost=True) + if not ret: + # Check line is flagged as none match + self.assertFalse(faildata.get('match', True), + "Line not matched when should have: %s:%i" % + (logFile.filename(), logFile.filelineno())) + elif ret: + # Check line is flagged to match + self.assertTrue(faildata.get('match', False), + "Line matched when shouldn't have: %s:%i" % + (logFile.filename(), logFile.filelineno())) + self.assertEqual(len(ret), 1) + # Verify timestamp and host as expected + host, time = ret[0] + self.assertEqual(host, faildata.get("host", None)) + self.assertEqual( + datetime.datetime.fromtimestamp(time), + datetime.datetime.strptime( + faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")) + + return testFilter + +for filter_ in os.listdir(os.path.join(TEST_FILES_DIR, "logs")): + if os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", filter_)): + setattr( + FilterSamplesRegex, + "testSampleRegexs%s" % filter_.upper(), + testSampleRegexsFactory(filter_)) From 09850d6ba52c7c1fbbe007cfa11348ca60dd7ee1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 14 Jul 2013 10:25:06 +1000 Subject: [PATCH 51/93] DOC: shorten example and provide clarifcation and spelling fixes --- README.Solaris | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/README.Solaris b/README.Solaris index 10a5f88c..86f56241 100644 --- a/README.Solaris +++ b/README.Solaris @@ -69,27 +69,10 @@ FAIL2BAN CONFIGURATION OPT: Create /etc/fail2ban/fail2ban.local containing: -# Fail2Ban main configuration file -# -# Comments: use '#' for comment lines and ';' (following a space) for inline comments -# -# Changes: in most of the cases you should not modify this -# file, but provide customizations in fail2ban.local file, e.g.: -# -# [Definition] -# loglevel = 4 +# Fail2Ban configuration file for logging fail2ban on Solaris # [Definition] -# Option: logtarget -# Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. -# Only one log target can be specified. -# If you change logtarget from the default value and you are -# using logrotate -- also adjust or disable rotation in the -# corresponding configuration file -# (e.g. /etc/logrotate.d/fail2ban on Debian systems) -# Values: STDOUT STDERR SYSLOG file Default: /var/log/fail2ban.log -# logtarget = /var/adm/fail2ban.log @@ -105,7 +88,7 @@ ignoreregex = for myuser from logpath = /var/adm/auth.log Set the sendmail dest address to something useful or drop the line to stop it spamming you. -Set 'myuser' to your username to avoid banning yourself or drop it. +Set 'myuser' to your username to avoid banning yourself or remove the line. START (OR RESTART) FAIL2BAN @@ -128,7 +111,7 @@ GOTCHAS AND FIXMES svcadm enable fail2ban * If svcs -xv says that fail2ban failed to start or svcs says it's in maintenance mode - chcek /var/svc/log/network-fail2ban:default.log for clues. + check /var/svc/log/network-fail2ban:default.log for clues. Check permissions on /var/adm, /var/adm/auth.log /var/adm/fail2ban.log and /var/run/fail2ban You may need to: From 94376bfbe1a7a097876c21582558425db500bf93 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 14 Jul 2013 11:15:45 +0100 Subject: [PATCH 52/93] TST: Handle lack of `json` library in python2.5 for samples test case --- fail2ban-testcases | 16 +++++++++++++--- testcases/samplestestcase.py | 8 +++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 278dd890..8ba697b3 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -27,17 +27,26 @@ __license__ = "GPL" import unittest, logging, sys, time, os +if sys.version_info >= (2, 6): + import json +else: + try: + import simplejson as json + except ImportError: + json = None + from common.version import version from testcases import banmanagertestcase from testcases import clientreadertestcase from testcases import failmanagertestcase from testcases import filtertestcase -from testcases import samplestestcase from testcases import servertestcase from testcases import datedetectortestcase from testcases import actiontestcase from testcases import sockettestcase from testcases import misctestcase +if json: + from testcases import samplestestcase from testcases.utils import FormatterWithTraceBack from server.mytime import MyTime @@ -171,8 +180,9 @@ tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) -# Filter Regex tests with sample logs -tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) +if json: + # Filter Regex tests with sample logs + tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) # # Extensive use-tests of different available filters backends diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 2bb73ca1..656bbffa 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -22,7 +22,13 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import unittest, os, fileinput, re, json, datetime +import unittest, sys, os, fileinput, re, datetime + +if sys.version_info >= (2, 6): + import json +else: + import simplejson as json + next = lambda x: x.next() from server.filter import Filter from client.filterreader import FilterReader From 1116f23151706c89e923d7e4db2b818591c27614 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 14 Jul 2013 18:19:16 +0100 Subject: [PATCH 53/93] TST: Sample log regex test now warns if no log for a filter Also checks that at least some tests are present --- fail2ban-testcases | 2 ++ testcases/samplestestcase.py | 44 +++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 8ba697b3..5aed6e43 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -183,6 +183,8 @@ tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) if json: # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) +else: + print "I: Skipping filter samples testing. No simplejson/json module" # # Extensive use-tests of different available filters backends diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 656bbffa..6b0e9ae0 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -22,7 +22,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import unittest, sys, os, fileinput, re, datetime +import unittest, sys, os, fileinput, re, datetime, inspect if sys.version_info >= (2, 6): import json @@ -46,6 +46,14 @@ class FilterSamplesRegex(unittest.TestCase): def tearDown(self): """Call after every test case.""" + def testFiltersPresent(self): + """Check to ensure some tests exist""" + self.assertTrue( + len([test for test in inspect.getmembers(self) + if test[0].startswith('testSampleRegexs')]) + >= 10, + "Expected more FilterSampleRegexs tests") + def testSampleRegexsFactory(name): def testFilter(self): @@ -58,8 +66,22 @@ def testSampleRegexsFactory(name): if opt[2] == "addfailregex": self.filter.addFailRegex(opt[3]) + if not self.filter.getFailRegex(): + # No fail regexs set: likely just common file for includes. + return + + # TODO: Remove exception handling once sample logs obtained for all + try: + self.assertTrue( + os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), + "No sample log file available for '%s' filter" % name) + except AssertionError: + print "I: No sample log file available for '%s' filter" % name + return + logFile = fileinput.FileInput( os.path.join(TEST_FILES_DIR, "logs", name)) + for line in logFile: jsonREMatch = re.match("^# ?failJSON:(.+)$", line) if jsonREMatch: @@ -78,13 +100,13 @@ def testSampleRegexsFactory(name): if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True), - "Line not matched when should have: %s:%i" % - (logFile.filename(), logFile.filelineno())) + "Line not matched when should have: %s:%i %r" % + (logFile.filename(), logFile.filelineno(), line)) elif ret: # Check line is flagged to match self.assertTrue(faildata.get('match', False), - "Line matched when shouldn't have: %s:%i" % - (logFile.filename(), logFile.filelineno())) + "Line matched when shouldn't have: %s:%i %r" % + (logFile.filename(), logFile.filelineno(), line)) self.assertEqual(len(ret), 1) # Verify timestamp and host as expected host, time = ret[0] @@ -96,9 +118,9 @@ def testSampleRegexsFactory(name): return testFilter -for filter_ in os.listdir(os.path.join(TEST_FILES_DIR, "logs")): - if os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", filter_)): - setattr( - FilterSamplesRegex, - "testSampleRegexs%s" % filter_.upper(), - testSampleRegexsFactory(filter_)) +for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")): + filterName = filter_.rpartition(".")[0] + setattr( + FilterSamplesRegex, + "testSampleRegexs%s" % filterName.upper(), + testSampleRegexsFactory(filterName)) From 2c8747cc76e88a58deb93d94fe39e51eaa9b8852 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Jul 2013 18:40:32 +0100 Subject: [PATCH 54/93] BF: fail2ban-regex date detector template hits count now correct closes #295 --- fail2ban-regex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index 78716a43..e75cb2a3 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -46,6 +46,7 @@ from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException +from server.datedetector import DateDetector from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. @@ -172,6 +173,8 @@ class Fail2banRegex(object): self._ignoreregex = list() self._failregex = list() self._line_stats = LineStats() + self._dateDetector = DateDetector() + self._dateDetector.addDefaultTemplate() def readRegex(self, value, regextype): @@ -256,6 +259,9 @@ class Fail2banRegex(object): if line.startswith('# ') or not line.strip(): # skip comment and empty lines continue + + self._dateDetector.matchTime(line) + is_ignored = fail2banRegex.testIgnoreRegex(line) if is_ignored: self._line_stats.ignored_lines.append(line) @@ -312,7 +318,7 @@ class Fail2banRegex(object): print "\nDate template hits:" out = [] - for template in self._filter.dateDetector.getTemplates(): + for template in self._dateDetector.getTemplates(): if self._verbose or template.getHits(): out.append("[%d] %s" % (template.getHits(), template.getName())) pprint_list(out, "[# of hits] date format") From 5bd186b85495319f717bfc94b8eb8888003bff43 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 15 Jul 2013 13:52:42 -0400 Subject: [PATCH 55/93] ENH(minor): fail2ban-regex comment line doesn't have to have a space after leading # --- fail2ban-regex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index e75cb2a3..3326e328 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -256,7 +256,7 @@ class Fail2banRegex(object): def process(self, test_lines): for line in test_lines: - if line.startswith('# ') or not line.strip(): + if line.startswith('#') or not line.strip(): # skip comment and empty lines continue From 1a2b6442a069a0aa7f86ff21e4a9dee55b893c1b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Jul 2013 22:16:40 +0100 Subject: [PATCH 56/93] ENH+BF+TST: Filter now returns reference to failregex and ignoreregex This avoids duplication of code across fail2ban-regex and samples test cases. This also now more neatly resolves the issue of double counting date templates matches in fail2ban-regex. In addition, the samples test cases now also print a warning message that not all regexs have samples for them, with future plan to change this to an assertion. --- fail2ban-regex | 85 +++++++++++++++--------------------- server/datedetector.py | 1 + server/datetemplate.py | 5 ++- server/filter.py | 42 ++++++++++-------- testcases/samplestestcase.py | 21 +++++++-- 5 files changed, 79 insertions(+), 75 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 3326e328..2c050e0d 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -46,7 +46,6 @@ from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException -from server.datedetector import DateDetector from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. @@ -130,7 +129,7 @@ class RegexStat(object): return self._failregex def appendIP(self, value): - self._ipList.extend(value) + self._ipList.append(value) def getIPList(self): return self._ipList @@ -173,8 +172,6 @@ class Fail2banRegex(object): self._ignoreregex = list() self._failregex = list() self._line_stats = LineStats() - self._dateDetector = DateDetector() - self._dateDetector.addDefaultTemplate() def readRegex(self, value, regextype): @@ -204,53 +201,39 @@ class Fail2banRegex(object): regex_values = [RegexStat(value)] setattr(self, "_" + regex, regex_values) + for regex in regex_values: + getattr( + self._filter, + 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True def testIgnoreRegex(self, line): found = False - for regex in self._ignoreregex: - try: - self._filter.addIgnoreRegex(regex.getFailRegex()) - try: - ret = self._filter.ignoreLine(line) - if ret: - found = True - regex.inc() - except RegexException, e: - print e - return False - finally: - self._filter.delIgnoreRegex(0) + try: + ret = self._filter.ignoreLine(line) + if ret is not None: + found = True + regex = self._ignoreregex[ret].inc() + except RegexException, e: + print e + return False return found def testRegex(self, line): - found = False - for regex in self._ignoreregex: - self._filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self._failregex: - try: - self._filter.addFailRegex(regex.getFailRegex()) - try: - ret = self._filter.processLine(line) - if len(ret): - if found == True: - ret[0].append(True) - else: - found = True - ret[0].append(False) - regex.inc() - regex.appendIP(ret) - except RegexException, e: - print e - return False - except IndexError: - print "Sorry, but no found in regex" - return False - finally: - self._filter.delFailRegex(0) - for regex in self._ignoreregex: - self._filter.delIgnoreRegex(0) - return found + try: + ret = self._filter.processLine(line, checkAllRegex=True) + for match in ret: + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) + except RegexException, e: + print e + return False + except IndexError: + print "Sorry, but no found in regex" + return False + return len(ret) > 0 def process(self, test_lines): @@ -259,9 +242,6 @@ class Fail2banRegex(object): if line.startswith('#') or not line.strip(): # skip comment and empty lines continue - - self._dateDetector.matchTime(line) - is_ignored = fail2banRegex.testIgnoreRegex(line) if is_ignored: self._line_stats.ignored_lines.append(line) @@ -302,10 +282,13 @@ class Fail2banRegex(object): if self._verbose and len(failregex.getIPList()): for ip in failregex.getIPList(): - timeTuple = time.localtime(ip[1]) + timeTuple = time.localtime(ip[2]) timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - out.append(" %s %s%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "")) + out.append( + " %s %s%s" % ( + ip[1], + timeString, + ip[3] and " (multiple regex matched)" or "")) print "\n%s: %d total" % (title, total) pprint_list(out, " #) [# of hits] regular expression") @@ -318,7 +301,7 @@ class Fail2banRegex(object): print "\nDate template hits:" out = [] - for template in self._dateDetector.getTemplates(): + for template in self._filter.dateDetector.getTemplates(): if self._verbose or template.getHits(): out.append("[%d] %s" % (template.getHits(), template.getName())) pprint_list(out, "[# of hits] date format") diff --git a/server/datedetector.py b/server/datedetector.py index 0c8b4df2..0ed9e00a 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -174,6 +174,7 @@ class DateDetector: match = template.matchDate(line) if not match is None: logSys.debug("Matched time template %s" % template.getName()) + template.incHits() return match return None finally: diff --git a/server/datetemplate.py b/server/datetemplate.py index 8c49aa15..86eeee8e 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -59,11 +59,12 @@ class DateTemplate: def getHits(self): return self.__hits + + def incHits(self): + self.__hits += 1 def matchDate(self, line): dateMatch = self.__cRegex.search(line) - if not dateMatch is None: - self.__hits += 1 return dateMatch def getDate(self, line): diff --git a/server/filter.py b/server/filter.py index d0390e62..ae12968e 100644 --- a/server/filter.py +++ b/server/filter.py @@ -284,7 +284,7 @@ class Filter(JailThread): return False - def processLine(self, line, returnRawHost=False): + def processLine(self, line, returnRawHost=False, checkAllRegex=False): """Split the time portion from log msg and return findFailures on them """ try: @@ -306,14 +306,15 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine, returnRawHost) + return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager """ for element in self.processLine(line): - ip = element[0] - unixTime = element[1] + failregex = element[0] + ip = element[1] + unixTime = element[2] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): @@ -335,11 +336,11 @@ class Filter(JailThread): # @return: a boolean def ignoreLine(self, line): - for ignoreRegex in self.__ignoreRegex: + for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): ignoreRegex.search(line) if ignoreRegex.hasMatched(): - return True - return False + return ignoreRegexIndex + return None ## # Finds the failure in a line given split into time and log parts. @@ -348,18 +349,19 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, timeLine, logLine, returnRawHost=False): + def findFailure(self, timeLine, logLine, + returnRawHost=False, checkAllRegex=False): failList = list() # Checks if we must ignore this line. - if self.ignoreLine(logLine): + if self.ignoreLine(logLine) is not None: # The ignoreregex matched. Return. return failList + date = self.dateDetector.getUnixTime(timeLine) # Iterates over all the regular expressions. - for failRegex in self.__failRegex: + for failRegexIndex, failRegex in enumerate(self.__failRegex): failRegex.search(logLine) if failRegex.hasMatched(): # The failregex matched. - date = self.dateDetector.getUnixTime(timeLine) logSys.log(7, "Date: %r, message: %r", timeLine, logLine) if date is None: @@ -372,14 +374,16 @@ class Filter(JailThread): try: host = failRegex.getHost() if returnRawHost: - failList.append([host, date]) - break - ipMatch = DNSUtils.textToIp(host, self.__useDns) - if ipMatch: - for ip in ipMatch: - failList.append([ip, date]) - # We matched a regex, it is enough to stop. - break + failList.append([failRegexIndex, host, date]) + if not checkAllRegex: + break + else: + ipMatch = DNSUtils.textToIp(host, self.__useDns) + if ipMatch: + for ip in ipMatch: + failList.append([failRegexIndex, ip, date]) + if not checkAllRegex: + break except RegexException, e: # pragma: no cover - unsure if reachable logSys.error(e) return failList diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 6b0e9ae0..a52873b9 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -82,6 +82,7 @@ def testSampleRegexsFactory(name): logFile = fileinput.FileInput( os.path.join(TEST_FILES_DIR, "logs", name)) + regexsUsed = set() for line in logFile: jsonREMatch = re.match("^# ?failJSON:(.+)$", line) if jsonREMatch: @@ -96,7 +97,8 @@ def testSampleRegexsFactory(name): else: faildata = {} - ret = self.filter.processLine(line, returnRawHost=True) + ret = self.filter.processLine( + line, returnRawHost=True, checkAllRegex=True) if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True), @@ -107,15 +109,28 @@ def testSampleRegexsFactory(name): self.assertTrue(faildata.get('match', False), "Line matched when shouldn't have: %s:%i %r" % (logFile.filename(), logFile.filelineno(), line)) - self.assertEqual(len(ret), 1) + self.assertEqual(len(ret), 1, "Multiple regexs matched") # Verify timestamp and host as expected - host, time = ret[0] + failregex, host, time = ret[0] self.assertEqual(host, faildata.get("host", None)) self.assertEqual( datetime.datetime.fromtimestamp(time), datetime.datetime.strptime( faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")) + regexsUsed.add(failregex) + + # TODO: Remove exception handling once all regexs have samples + for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): + try: + self.assertTrue( + failRegexIndex in regexsUsed, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) + except AssertionError: + print "I: Regex for filter '%s' has no samples: %i: %r" % ( + name, failRegexIndex, failRegex) + return testFilter for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")): From 148cbd8d2a2d3cab6815fc100a4e635c0e6a10ed Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Jul 2013 12:47:41 -0400 Subject: [PATCH 57/93] ENH: heavier debugging -- log split date/log line even for no match. Log matching regex upon match --- server/failregex.py | 3 ++- server/filter.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index 957c77db..d302370b 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -50,7 +50,8 @@ class Regex: except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - + def __str__(self): + return "%s(%r)" % (self.__class__.__name__, self._regex) ## # Gets the regular expression. # diff --git a/server/filter.py b/server/filter.py index d0390e62..54efb619 100644 --- a/server/filter.py +++ b/server/filter.py @@ -294,7 +294,7 @@ class Filter(JailThread): l = line l = l.rstrip('\r\n') - logSys.log(5, "Working on line %r", l) + logSys.log(7, "Working on line %r", l) timeMatch = self.dateDetector.matchTime(l) if timeMatch: # Lets split into time part and log part of the line @@ -349,19 +349,20 @@ class Filter(JailThread): # @return a dict with IP and timestamp. def findFailure(self, timeLine, logLine, returnRawHost=False): + logSys.log(5, "Date: %r, message: %r", timeLine, logLine) failList = list() # Checks if we must ignore this line. if self.ignoreLine(logLine): # The ignoreregex matched. Return. + logSys.log(7, "Matched ignoreregex and was ignored") return failList # Iterates over all the regular expressions. for failRegex in self.__failRegex: failRegex.search(logLine) if failRegex.hasMatched(): # The failregex matched. + logSys.log(7, "Matched %s", failRegex) date = self.dateDetector.getUnixTime(timeLine) - logSys.log(7, "Date: %r, message: %r", - timeLine, logLine) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" From 8add63c733f997feb0a3dd321cdf205ce22c3f04 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Jul 2013 12:48:12 -0400 Subject: [PATCH 58/93] ENH: anchor roundcube-auth at the beginning as well --- config/filter.d/roundcube-auth.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 7b153f44..77d872fb 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = common.conf + [Definition] # Option: failregex @@ -13,7 +17,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = (FAILED login|Login failed) for .* from \s*$ +failregex = ^\s*(\[(\s\+[0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .* from \s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 0a02cfe9e8e65e68ec438bcd1c10fb7493f3e842 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Jul 2013 14:39:42 -0400 Subject: [PATCH 59/93] ENH: must end with alphanumeric \w (not a dot or a dash etc) Otherwise regexp might swallow period in the sentence right after the address. I have decided to enforce alphanumeric instead of switching to non-greedy +? ... because I think it is closer to what we actually want here --- server/failregex.py | 2 +- testcases/servertestcase.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index d302370b..bfa50575 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -41,7 +41,7 @@ class Regex: self._matchCache = None # Perform shortcuts expansions. # Replace "" with default regular expression for host. - regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]+)") + regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]*\w)") if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 0a5593e3..07cc373c 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -334,9 +334,9 @@ class Transmitter(TransmitterBase): "failed attempt from again", ], [ - "user john at (?:::f{4,6}:)?(?P[\w\-.^_]+)", - "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", - "failed attempt from (?:::f{4,6}:)?(?P[\w\-.^_]+) again", + "user john at (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", + "failed attempt from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w) again", ], self.jailName ) @@ -359,7 +359,7 @@ class Transmitter(TransmitterBase): ], [ "user john", - "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", "Dont match me!", ], self.jailName From f6a8a04cf34b11e317ac8b90970f9ae6518980e9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Jul 2013 14:45:59 -0400 Subject: [PATCH 60/93] ENH: roundcube-auth - adopt for current format with trailing error message. thanks @kwirk for the review/feedback I also used non-greedy .*? for the login portion since not sure if space could be there and trying to minimize possibility of reacting on injected "from " somewhere within the trailing .* --- config/filter.d/roundcube-auth.conf | 2 +- testcases/files/logs/roundcube-auth | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 77d872fb..fe669f66 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -17,7 +17,7 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^\s*(\[(\s\+[0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .* from \s*$ +failregex = ^\s*(\[(\s\+[0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index dec8051e..7c16efbd 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -2,3 +2,5 @@ [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 # failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" } May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) From 90ec82669c696894dfeb0d1c71dc01f415be15e7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 16 Jul 2013 15:10:41 -0400 Subject: [PATCH 61/93] DOC: changelog entries for preceeding changes --- ChangeLog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 16c93db1..a6a6ab23 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,7 +29,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Daniel Black * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across all platforms to ensure permissions are the same before and after a ban - - closes gh-266 + closes gh-266 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added @@ -51,6 +51,8 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests * fail2ban-client -- log to standard error. Closes gh-264 * Fail to configure if not a single log file was found for an enabled jail. Closes gh-63 + * is now enforced to end with an alphanumeric + * filter.d/roundcube-auth.conf -- anchored version Alexander Dietrich * action.d/sendmail-common.conf -- added common sendmail settings file and made the sender display name configurable From c2bdfefb626899c7c249dfc37e93c994377b568f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 16 Jul 2013 20:58:44 +0100 Subject: [PATCH 62/93] DOC: Comment to fail2ban-regex - flagging lines matched multiple regexs --- fail2ban-regex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban-regex b/fail2ban-regex index 2c050e0d..e19b1bc9 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -223,6 +223,8 @@ class Fail2banRegex(object): try: ret = self._filter.processLine(line, checkAllRegex=True) for match in ret: + # Append True/False flag depending if line was matched by + # more than one regex match.append(len(ret)>1) regex = self._failregex[match[0]] regex.inc() From ba29f6bef3cb1757155e7a2bae475eac6afb3aa0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 16 Jul 2013 21:11:10 +0100 Subject: [PATCH 63/93] DOC: Update doc in reference to changes for sample testcases --- DEVELOP | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DEVELOP b/DEVELOP index 2d7a043b..b0f7b47e 100644 --- a/DEVELOP +++ b/DEVELOP @@ -39,8 +39,9 @@ Filters * Include sample logs with 1.2.3.4 used for IP addresses and example.com/example.org used for DNS names -* Ensure ./fail2ban-regex testcases/files/logs/{samplelog} config/filter.d/{filter}.conf - has matches for EVERY regex +* Ensure sample log is provided in testcases/files/logs/ with same name as the + filter. Each log line should include match meta data for time & IP above + every line (see other sample log files for examples) * Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if \d+ is sufficient * Use the functionality of regexs http://docs.python.org/2/library/re.html From 60aa44f42d34c050a20228462bf6bf380752b057 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Jul 2013 14:43:41 -0400 Subject: [PATCH 64/93] DOC: added a note on commit comments for merges with Conflicts --- DEVELOP | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEVELOP b/DEVELOP index b0f7b47e..61248393 100644 --- a/DEVELOP +++ b/DEVELOP @@ -253,6 +253,10 @@ Use the text "closes #333"/"resolves #333 "/"fixes #333" where 333 represents an issue that is closed. Other text and details in link below. See: https://help.github.com/articles/closing-issues-via-commit-messages +If merge resulted in conflicts, clarify what changes were done to +corresponding files in the 'Conflicts:' section of the merge commit +message. See e.g. https://github.com/fail2ban/fail2ban/commit/f5a8a8ac + Adding Actions -------------- From 2a3a6273223aa98c12d2071adb385ff073252da9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Jul 2013 22:17:14 +0100 Subject: [PATCH 65/93] TST: Add sample for sieve regex --- testcases/files/logs/sieve | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/sieve diff --git a/testcases/files/logs/sieve b/testcases/files/logs/sieve new file mode 100644 index 00000000..5cc19673 --- /dev/null +++ b/testcases/files/logs/sieve @@ -0,0 +1,2 @@ +# failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" } +Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure From a012b5411715f3a478cd7b19acd70e652044d625 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Jul 2013 22:17:31 +0100 Subject: [PATCH 66/93] TST: Add additional postfix filter sample --- testcases/files/logs/postfix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 50e3a360..e4d07b58 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -2,3 +2,5 @@ # and https://github.com/fail2ban/fail2ban/issues/126 # failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" } Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= +# failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" } +Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 : Relay access denied; from= to= proto=SMTP helo=<198.51.100.17> From eb2f0c927257120dfc32d2450fd63f1962f38821 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:31:21 +0100 Subject: [PATCH 67/93] ENH: Improve postfix regex and add more samples --- config/filter.d/postfix.conf | 13 +++++++++++-- testcases/files/logs/postfix | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f92c3619..b70f0377 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = postfix/smtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,8 +22,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = reject: RCPT from (.*)\[\]: 554 - reject: RCPT from (.*)\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index e4d07b58..172bfca4 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -4,3 +4,7 @@ Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= # failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" } Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 : Relay access denied; from= to= proto=SMTP helo=<198.51.100.17> +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.65" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 : Helo command rejected: match bad.domain; from= to= proto=SMTP helo= +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> From 83a80a29ea51d1cb2dc1ce45a33b1d44a8619c9c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:34:00 +0100 Subject: [PATCH 68/93] ENH: Improve couriersmtp and add sample logs --- config/filter.d/couriersmtp.conf | 11 ++++++++++- testcases/files/logs/couriersmtp | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testcases/files/logs/couriersmtp diff --git a/config/filter.d/couriersmtp.conf b/config/filter.d/couriersmtp.conf index 6c0cf5ff..65ffa5d7 100644 --- a/config/filter.d/couriersmtp.conf +++ b/config/filter.d/couriersmtp.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = courieresmtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,7 +22,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = error,relay=,.*550 User unknown +failregex = ^%(__prefix_line)serror,relay=,.*: 550 User unknown\.$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/couriersmtp b/testcases/files/logs/couriersmtp new file mode 100644 index 00000000..212df3b4 --- /dev/null +++ b/testcases/files/logs/couriersmtp @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-10T03:47:57", "match": true , "host": "1.2.3.4" } +Apr 10 03:47:57 web courieresmtpd: error,relay=::ffff:1.2.3.4,ident=tmf,from=,to=: 550 User unknown. +# failJSON: { "time": "2005-07-06T03:42:28", "match": true , "host": "1.2.3.4" } +Jul 6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=: 550 User unknown. +# failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } +Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. +# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. From bd175f026737d66e7110868fb50b3760ff75e087 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:38:29 +0100 Subject: [PATCH 69/93] ENH: Improve cyrus-imap regex and add sample log file --- config/filter.d/cyrus-imap.conf | 16 ++++++++++++---- testcases/files/logs/cyrus-imap | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 testcases/files/logs/cyrus-imap diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index 758f75de..ae7bf6bd 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:cyrus/)?(?:imapd?|pop3d?) + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,10 +22,9 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = : badlogin: .*\[\] plaintext .*SASL\(-13\): authentication failure: checkpass failed$ - : badlogin: .*\[\] LOGIN \[SASL\(-13\): authentication failure: checkpass failed\]$ - : badlogin: .*\[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ - : badlogin: .*\[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ \[\] (?:plaintext|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ + ^%(__prefix_line)sbadlogin: \S+ \[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ + ^%(__prefix_line)sbadlogin: \S+ \[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/cyrus-imap b/testcases/files/logs/cyrus-imap new file mode 100644 index 00000000..5886938e --- /dev/null +++ b/testcases/files/logs/cyrus-imap @@ -0,0 +1,10 @@ +# failJSON: { "time": "2005-01-04T21:51:05", "match": true , "host": "127.0.0.1" } +Jan 4 21:51:05 hostname cyrus/imap[5355]: badlogin: localhost.localdomain [127.0.0.1] plaintext cyrus@localdomain SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "198.51.100.23" } +Feb 20 17:23:32 domain cyrus/pop3[18635]: badlogin: localhost [198.51.100.23] plaintext administrator SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "1.2.3.4" } +Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail0001 SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-06-08T18:11:13", "match": true , "host": "198.51.100.45" } +Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated] +# failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" } +Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response] From 57a6c11260c835b221b88edf8821948128953bed Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 15:53:18 +0100 Subject: [PATCH 70/93] ENH: Improve courierlogin regex and add sample logs --- config/filter.d/courierlogin.conf | 11 ++++++++++- testcases/files/logs/courierlogin | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testcases/files/logs/courierlogin diff --git a/config/filter.d/courierlogin.conf b/config/filter.d/courierlogin.conf index 20731e5d..f096325e 100644 --- a/config/filter.d/courierlogin.conf +++ b/config/filter.d/courierlogin.conf @@ -5,8 +5,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -14,7 +23,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = LOGIN FAILED, .*, ip=\[\]$ +failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/courierlogin b/testcases/files/logs/courierlogin new file mode 100644 index 00000000..e3d0d8c3 --- /dev/null +++ b/testcases/files/logs/courierlogin @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-23T21:59:01", "match": true , "host": "1.2.3.4" } +Apr 23 21:59:01 dns2 imapd: LOGIN FAILED, user=sales@example.com, ip=[::ffff:1.2.3.4] +# failJSON: { "time": "2005-04-23T21:59:38", "match": true , "host": "198.51.100.76" } +Apr 23 21:59:38 dns2 pop3d: LOGIN FAILED, user=info@example.com, ip=[::ffff:198.51.100.76] +# failJSON: { "time": "2004-11-13T08:11:53", "match": true , "host": "198.51.100.33" } +Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff:198.51.100.33] +# failJSON: { "time": "2005-04-17T19:17:11", "match": true , "host": "1.2.3.4" } +Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4] From ab671b0b1a2c6beec54f8cb585716513210db62e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 16:34:24 +0100 Subject: [PATCH 71/93] ENH: Improve wuftpd failregex, drop duplicate pam regex and add sample For wu-ftpd configured to use pam, the pam filter used be used, as regex is more robust. --- config/filter.d/wuftpd.conf | 11 +++++++++-- testcases/files/logs/wuftpd | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 3351d258..5c1cbb58 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -4,14 +4,21 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] +_daemon = wu-ftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = wu-ftpd(?:\[\d+\])?:\s+\(pam_unix\)\s+authentication failure.* rhost=$ - wu-ftpd(?:\[\d+\])?: *failed login from .*\[\] *$ +failregex = ^%(__prefix_line)sfailed login from \S+ \[\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/wuftpd b/testcases/files/logs/wuftpd index 22ac0303..bbb816cc 100644 --- a/testcases/files/logs/wuftpd +++ b/testcases/files/logs/wuftpd @@ -1,3 +1,5 @@ # This login line is from syslog # failJSON: { "time": "2004-10-06T09:59:26", "match": true , "host": "202.108.145.173" } Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] +# failJSON: { "time": "2004-10-11T16:45:07", "match": true , "host": "198.51.100.71" } +Oct 11 16:45:07 ubuntu wu-ftpd[2360]: failed login from example.com [198.51.100.71] From d791ba12bab3cbf7c1d804cff91b6dc30a254bcc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 16:54:28 +0100 Subject: [PATCH 72/93] TST: Add sample log for dropbear filter --- testcases/files/logs/dropbear | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 testcases/files/logs/dropbear diff --git a/testcases/files/logs/dropbear b/testcases/files/logs/dropbear new file mode 100644 index 00000000..5a2e197a --- /dev/null +++ b/testcases/files/logs/dropbear @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 From 534be189dcd43f93182ee14f32ce78e67e5c5084 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 17:26:09 +0100 Subject: [PATCH 73/93] ENH: Improve sieve regex and add sample line --- config/filter.d/sieve.conf | 11 ++++++++++- testcases/files/logs/sieve | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sieve.conf b/config/filter.d/sieve.conf index 866b4228..83608fa6 100644 --- a/config/filter.d/sieve.conf +++ b/config/filter.d/sieve.conf @@ -4,15 +4,24 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_deamon = (?:cyrus/)?(?:tim)?sieved? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can # be used for standard IP/hostname matching. # Values: TEXT # -failregex = : badlogin: .*\[\] (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failure$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:LOGIN|PLAIN|OTP|(?:CRAM|DIGEST)-MD5) authentication failure$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/sieve b/testcases/files/logs/sieve index 5cc19673..770d17ad 100644 --- a/testcases/files/logs/sieve +++ b/testcases/files/logs/sieve @@ -1,2 +1,6 @@ # failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" } Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2005-07-18T17:21:58", "match": true , "host": "1.2.3.4" } +Jul 18 17:21:58 ophelia cyrus/timsieved[12305]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2004-09-25T22:07:38", "match": true , "host": "1.2.3.4" } +Sep 25 22:07:38 web9 timsieved[21040]: badlogin: web4[1.2.3.4] OTP authentication failure From a11f91b83506289d03c4df2bf3c5ea3cad27f665 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Jul 2013 17:28:28 +0100 Subject: [PATCH 74/93] ENH: Improve cyrus-imap regex and add extra sample line --- config/filter.d/cyrus-imap.conf | 5 ++--- testcases/files/logs/cyrus-imap | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index ae7bf6bd..b63fad8f 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -22,9 +22,8 @@ _daemon = (?:cyrus/)?(?:imapd?|pop3d?) # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ \[\] (?:plaintext|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ - ^%(__prefix_line)sbadlogin: \S+ \[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ - ^%(__prefix_line)sbadlogin: \S+ \[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:plaintext|PLAIN|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ + ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:(?:CRAM|DIGEST)-MD5|NTLM|OTP) \[SASL\(-13\): authentication failure: .*\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/cyrus-imap b/testcases/files/logs/cyrus-imap index 5886938e..9bf271f6 100644 --- a/testcases/files/logs/cyrus-imap +++ b/testcases/files/logs/cyrus-imap @@ -8,3 +8,6 @@ Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated] # failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" } Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response] +# failJSON: { "time": "2004-12-30T16:03:27", "match": true , "host": "1.2.3.4" } +Dec 30 16:03:27 somehost imapd[2517]: badlogin: local-somehost[1.2.3.4] OTP [SASL(-13): authentication failure: External SSF not good enough] + From e7b7815de33446a8705dac69326bb167fcd302ec Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:22:44 +0100 Subject: [PATCH 75/93] TST: Add additional sshd sample logs --- testcases/files/logs/sshd | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index ff97c5a5..75854774 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -77,3 +77,11 @@ Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error #11 https://github.com/fail2ban/fail2ban/issues/267 There might be no colon after [daemon] # failJSON: { "time": "2005-06-25T23:53:34", "match": true , "host": "1.2.3.4" } Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers +# failJSON: { "time": "2004-12-12T20:04:39", "match": true , "host": "10.215.4.227" } +Dec 12 20:04:39 aragorn sshd[1328]: error: PAM: User not known to the underlying authentication module for illegal user kernelitshell from 10.215.4.227 +# failJSON: { "time": "2005-03-26T04:56:27", "match": true , "host": "example.com" } +Mar 26 04:56:27 angel sshd[9739]: User allena from example.com not allowed because not in any group +# failJSON: { "time": "2005-02-07T16:01:07", "match": true , "host": "192.51.100.54" } +Feb 7 16:01:07 linux-m899 sshd[5106]: User root from 192.51.100.54 not allowed because a group is listed in DenyGroups +# failJSON: { "time": "2005-01-05T11:15:05", "match": true , "host": "10.0.0.40" } +Jan 5 11:15:05 NAS sshd[1966]: User root from 10.0.0.40 not allowed because none of user's groups are listed in AllowGroups From b5ffbced37b3000aae229f0bcac88029646be4b2 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:31:32 +0100 Subject: [PATCH 76/93] TST: Sample test cases now handle ignoreregex and add recidive samples --- testcases/files/logs/recidive | 6 ++++++ testcases/samplestestcase.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 testcases/files/logs/recidive diff --git a/testcases/files/logs/recidive b/testcases/files/logs/recidive new file mode 100644 index 00000000..cf6df933 --- /dev/null +++ b/testcases/files/logs/recidive @@ -0,0 +1,6 @@ +# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [sendmail] Ban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 16:07:31,183 fail2ban.actions: WARNING [sendmail] Unban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [recidive] Ban 1.2.3.4 diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index a52873b9..dd478fbf 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -65,6 +65,8 @@ def testSampleRegexsFactory(name): for opt in filterConf.convert(): if opt[2] == "addfailregex": self.filter.addFailRegex(opt[3]) + elif opt[2] == "addignoreregex": + self.filter.addIgnoreRegex(opt[3]) if not self.filter.getFailRegex(): # No fail regexs set: likely just common file for includes. From 4033857f63cdb5702bf15e436a7360880d8a1676 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 15:44:09 +0100 Subject: [PATCH 77/93] ENH: Improve xinetd-fail regex and add sample logs --- config/filter.d/xinetd-fail.conf | 13 +++++++++++-- testcases/files/logs/xinetd-fail | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 testcases/files/logs/xinetd-fail diff --git a/config/filter.d/xinetd-fail.conf b/config/filter.d/xinetd-fail.conf index 4ff5bfde..253ce15d 100644 --- a/config/filter.d/xinetd-fail.conf +++ b/config/filter.d/xinetd-fail.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = xinetd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -19,8 +28,8 @@ # load => xinetd: max_load (temporary problem) # -failregex = xinetd(?:\[\d{1,5}\])?: FAIL: \S+ address from=$ - xinetd(?:\[\d{1,5}\])?: FAIL: \S+ libwrap from=$ +failregex = ^%(__prefix_line)sFAIL: \S+ address from=$ + ^%(__prefix_line)sFAIL: \S+ libwrap from=$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/xinetd-fail b/testcases/files/logs/xinetd-fail new file mode 100644 index 00000000..8545a067 --- /dev/null +++ b/testcases/files/logs/xinetd-fail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-05-15T17:38:49", "match": true , "host": "198.51.100.169" } +May 15 17:38:49 boo xinetd[16256]: FAIL: telnet address from=198.51.100.169 +# failJSON: { "time": "2005-08-03T14:38:49", "match": true , "host": "198.51.100.223" } +Aug 3 14:38:49 backup xinetd[31234]: FAIL: amanda libwrap from=198.51.100.223 From 8b9bafda7988d007b8e02dc21e17ea8fd6b07c30 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 16:31:11 +0100 Subject: [PATCH 78/93] ENH: Change lighttpd-fastcgi to suhosin, and improve regex and samples suhosin is hardened php implmentation, which will log the alerts (as seen in samples) to stderr, which is picked up by fastcgi webserver (e.g. lighttpd, apache, nginx) --- MANIFEST | 2 +- .../{lighttpd-fastcgi.conf => suhosin.conf} | 3 ++- config/jail.conf | 16 +++------------- testcases/files/logs/suhosin | 4 ++++ 4 files changed, 10 insertions(+), 15 deletions(-) rename config/filter.d/{lighttpd-fastcgi.conf => suhosin.conf} (67%) create mode 100644 testcases/files/logs/suhosin diff --git a/MANIFEST b/MANIFEST index 5491c7d5..f637dca0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -101,7 +101,7 @@ config/filter.d/couriersmtp.conf config/filter.d/cyrus-imap.conf config/filter.d/exim.conf config/filter.d/gssftpd.conf -config/filter.d/lighttpd-fastcgi.conf +config/filter.d/suhosin.conf config/filter.d/named-refused.conf config/filter.d/postfix.conf config/filter.d/proftpd.conf diff --git a/config/filter.d/lighttpd-fastcgi.conf b/config/filter.d/suhosin.conf similarity index 67% rename from config/filter.d/lighttpd-fastcgi.conf rename to config/filter.d/suhosin.conf index 1c6e3fce..c79c157f 100644 --- a/config/filter.d/lighttpd-fastcgi.conf +++ b/config/filter.d/suhosin.conf @@ -9,7 +9,8 @@ # Notes.: regex to match ALERTS as notified by lighttpd's FastCGI Module # Values: TEXT # -failregex = .*ALERT\ -\ .*attacker\ \'\' +# https://github.com/stefanesser/suhosin/blob/1fba865ab73cc98a3109f88d85eb82c1bfc29b37/log.c#L161 +failregex = ALERT - .* \(attacker '', file '.*'(?:, line \d+)?\)$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/jail.conf b/config/jail.conf index e3b92038..640a5874 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -256,25 +256,15 @@ filter = php-url-fopen logpath = /var/www/*/logs/access_log maxretry = 1 -# A simple PHP-fastcgi jail which works with lighttpd. -# If you run a lighttpd server, then you probably will -# find these kinds of messages in your error_log: -# ALERT – tried to register forbidden variable ‘GLOBALS’ -# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') -# This jail would block the IP 1.2.3.4. - -[lighttpd-fastcgi] +[suhosin] enabled = false -filter = lighttpd-fastcgi -action = iptables-multiport[name=lighttpd-fastcgi, port="http,https"] +filter = suhosin +action = iptables-multiport[name=suhosin, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 -# Same as above for mod_auth -# It catches wrong authentications - [lighttpd-auth] enabled = false diff --git a/testcases/files/logs/suhosin b/testcases/files/logs/suhosin new file mode 100644 index 00000000..90ed7bf1 --- /dev/null +++ b/testcases/files/logs/suhosin @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-03-11T22:52:12", "match": true , "host": "198.51.100.167" } +Mar 11 22:52:12 lighttpd[53690]: (mod_fastcgi.c.2676) FastCGI-stderr: ALERT - configured request variable name length limit exceeded - dropped variable 'upqchi07vFfAFuBjnIKGIwiLrHo3Vt68T3yqvhQu2TqetQ78roy7Q6bpTfDUtYFR593/MA' (attacker '198.51.100.167', file '/usr/local/captiveportal/index.php') +# failJSON: { "time": "2005-02-26T22:52:29", "match": true , "host": "198.51.100.77" } +Feb 26 22:52:29 host suhosin[9636]: ALERT - script tried to increase memory_limit to 268435456 bytes which is above the allowed value (attacker '198.51.100.77', file '/var/www/wordpress/wp-admin/includes/image.php', line 161) From e59a4960a3bea0e90fcb01ffcc03092ebf1c10f3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 16:48:12 +0100 Subject: [PATCH 79/93] TST: Add additional sample log line for apache-noscript --- testcases/files/logs/apache-noscript | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/files/logs/apache-noscript b/testcases/files/logs/apache-noscript index e08b3468..172bb510 100644 --- a/testcases/files/logs/apache-noscript +++ b/testcases/files/logs/apache-noscript @@ -1,2 +1,4 @@ # failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" } [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat +# failJSON: { "time": "2005-07-22T06:48:30", "match": true , "host": "198.51.100.86" } +[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php From cf1e5bdbc223384340b2457b9eba08524f924ba8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 22:03:49 +0100 Subject: [PATCH 80/93] ENH: Tweak proftpd regex and add sample logs Needed to add optional ":" post __pid_re, and for consistency, decided to make use of __prefix_line instead which includes this. --- config/filter.d/proftpd.conf | 10 ++++++---- testcases/files/logs/proftpd | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 13080fcc..d17658ce 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -13,6 +13,8 @@ before = common.conf [Definition] +_deamon = proftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -20,10 +22,10 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ +failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index 404ebfaf..9687d992 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -10,6 +10,7 @@ Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting. Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. # failJSON: { "time": "2005-06-14T00:09:59", "match": true , "host": "59.167.242.100" } Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 - - - +# failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } +May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 From 37f240bef00febcbe2711014ba1c4d53dab5070c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Jul 2013 22:13:37 +0100 Subject: [PATCH 81/93] TST: Add sample log for php-url-fopen filter --- testcases/files/logs/php-url-fopen | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/php-url-fopen diff --git a/testcases/files/logs/php-url-fopen b/testcases/files/logs/php-url-fopen new file mode 100644 index 00000000..f119a928 --- /dev/null +++ b/testcases/files/logs/php-url-fopen @@ -0,0 +1,2 @@ +# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" } +66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" From b5639a8672b3ace9c3e761d05fc76b50c29d3b17 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 11:55:09 +0100 Subject: [PATCH 82/93] ENH: Simplify cyrus-imap filter fail regex --- config/filter.d/cyrus-imap.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index b63fad8f..0ace92c1 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -22,8 +22,7 @@ _daemon = (?:cyrus/)?(?:imapd?|pop3d?) # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:plaintext|PLAIN|LOGIN) .* \[?SASL\(-13\): authentication failure: checkpass failed\]?$ - ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:(?:CRAM|DIGEST)-MD5|NTLM|OTP) \[SASL\(-13\): authentication failure: .*\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ .*?\[?SASL\(-13\): authentication failure: .*\]?$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 2f4aaa9fb9a26b89149a5a7e831f9a7bd765d80f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 12:01:09 +0100 Subject: [PATCH 83/93] ENH: Simplify sieve filter failregex --- config/filter.d/sieve.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sieve.conf b/config/filter.d/sieve.conf index 83608fa6..b2af6774 100644 --- a/config/filter.d/sieve.conf +++ b/config/filter.d/sieve.conf @@ -21,7 +21,7 @@ _deamon = (?:cyrus/)?(?:tim)?sieved? # be used for standard IP/hostname matching. # Values: TEXT # -failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] (?:LOGIN|PLAIN|OTP|(?:CRAM|DIGEST)-MD5) authentication failure$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ authentication failure$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From f7d8e6873885ec37843a346171a544dc6a4e4600 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 12:32:29 +0100 Subject: [PATCH 84/93] TST: Add apache-badbots sample log --- testcases/files/logs/apache-badbots | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/apache-badbots diff --git a/testcases/files/logs/apache-badbots b/testcases/files/logs/apache-badbots new file mode 100644 index 00000000..35669252 --- /dev/null +++ b/testcases/files/logs/apache-badbots @@ -0,0 +1,2 @@ +# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } +1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" From 5437f5fe908d3a95926d058e0fb4da9b4ab829d6 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:02:53 +0100 Subject: [PATCH 85/93] TST: Add gssftpd sample log --- testcases/files/logs/gssftpd | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/gssftpd diff --git a/testcases/files/logs/gssftpd b/testcases/files/logs/gssftpd new file mode 100644 index 00000000..58fda7dc --- /dev/null +++ b/testcases/files/logs/gssftpd @@ -0,0 +1,2 @@ +# failJSON: { "time": "2005-01-22T18:09:46", "match": true , "host": "198.51.100.23" } +Jan 22 18:09:46 host ftpd[132]: repeated login failures from 198.51.100.23 (example.com) From 1c7d28d1ea8e79dd8bd11f7899a2a4d4be1f4830 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:03:14 +0100 Subject: [PATCH 86/93] TST: Add qmail sample log --- testcases/files/logs/qmail | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 testcases/files/logs/qmail diff --git a/testcases/files/logs/qmail b/testcases/files/logs/qmail new file mode 100644 index 00000000..63cad4cc --- /dev/null +++ b/testcases/files/logs/qmail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2004-09-06T07:33:33", "match": true , "host": "198.51.100.77" } +Sep 6 07:33:33 sd6 qmail: 1157520813.485077 rblsmtpd: 198.51.100.77 pid 19597 sbl-xbl.spamhaus.org: 451 http://www.spamhaus.org/query/bl?ip=198.51.100.77 +# failJSON: { "time": "2004-09-06T07:18:29", "match": true , "host": "198.51.100.54" } +Sep 6 07:18:29 sd6 qmail: 1157519909.633171 qmail-smtpd: 421 badiprbl: ip 198.51.100.54 rbl: example.com From bf021ebd97719c9a1b7436366f4379f3d86d71ed Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:05:17 +0100 Subject: [PATCH 87/93] TST: Mandate that all filters and each regex has sample log entry --- testcases/samplestestcase.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index dd478fbf..54afc864 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -72,14 +72,9 @@ def testSampleRegexsFactory(name): # No fail regexs set: likely just common file for includes. return - # TODO: Remove exception handling once sample logs obtained for all - try: - self.assertTrue( - os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), - "No sample log file available for '%s' filter" % name) - except AssertionError: - print "I: No sample log file available for '%s' filter" % name - return + self.assertTrue( + os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), + "No sample log file available for '%s' filter" % name) logFile = fileinput.FileInput( os.path.join(TEST_FILES_DIR, "logs", name)) @@ -124,14 +119,10 @@ def testSampleRegexsFactory(name): # TODO: Remove exception handling once all regexs have samples for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): - try: - self.assertTrue( - failRegexIndex in regexsUsed, - "Regex for filter '%s' has no samples: %i: %r" % - (name, failRegexIndex, failRegex)) - except AssertionError: - print "I: Regex for filter '%s' has no samples: %i: %r" % ( - name, failRegexIndex, failRegex) + self.assertTrue( + failRegexIndex in regexsUsed, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) return testFilter From 11100a2fc0c65677f3e5d7b065c012aa37cbd547 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 26 Jul 2013 17:20:05 +0100 Subject: [PATCH 88/93] DOC: Changelog additions for regex changes --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index a6a6ab23..237ef672 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,10 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Steven Hiscocks * filter.d/dovecot - Addition of session, time values and possible blank user + * filter.d/{courier{login,smtp},proftpd,sieve,wuftpd,xinetd} - General + regex impovements + * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which + will require changing in jail.{conf,local} if using this filter. ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- From 9002de069ebe13a59ee9bada65763120b4290a4f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 9 Aug 2013 18:54:08 +0100 Subject: [PATCH 89/93] ENH: Improve proftpd regex. Taken from @yarikoptic comment: https://github.com/fail2ban/fail2ban/pull/303#discussion_r5687500 --- config/filter.d/proftpd.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index d17658ce..872ab0ca 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -22,8 +22,10 @@ _deamon = proftpd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # + +__suffix_failed_login = (User not authorized|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ From 53e16e07addb827ab11e608cd4204f0513f09ec4 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 9 Aug 2013 19:04:26 +0100 Subject: [PATCH 90/93] ENH: Minor tweak on previous commit proftpd regex changes --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 872ab0ca..d3120216 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -23,7 +23,7 @@ _deamon = proftpd # Values: TEXT # -__suffix_failed_login = (User not authorized|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ From 61d43608ae4d6a142469c91ec4c0611cf117a08d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 18:32:07 +1000 Subject: [PATCH 91/93] ENH: filter.d/postfix - add filter for VRFY. Closes gh-322 --- ChangeLog | 3 +++ config/filter.d/postfix.conf | 1 + testcases/files/logs/postfix | 2 ++ 3 files changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 237ef672..0252e9fd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -63,6 +63,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests regex impovements * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which will require changing in jail.{conf,local} if using this filter. + Zurd + * filter.d/postfix - add filter for VRFY failures. closes gh-322. + ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index b70f0377..da981733 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -24,6 +24,7 @@ _daemon = postfix/smtpd # failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 172bfca4..122ad8e5 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -8,3 +8,5 @@ Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-e Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 : Helo command rejected: match bad.domain; from= to= proto=SMTP helo= # failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" } Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> +# failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" } +Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab From ed42b08789525afb1138b9b0508aad22cdfb4839 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 21:22:17 +1000 Subject: [PATCH 92/93] TST: merge dropbear log samples --- testcases/files/logs/dropbear | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testcases/files/logs/dropbear b/testcases/files/logs/dropbear index 5a2e197a..139292d7 100644 --- a/testcases/files/logs/dropbear +++ b/testcases/files/logs/dropbear @@ -2,3 +2,8 @@ Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 # failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 + From ea7cba420598fa2a38b2ce01fc9737cba5c03eaa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 19 Aug 2013 21:42:43 +1000 Subject: [PATCH 93/93] ENH: trailing space as per discussion on gh-303 --- config/filter.d/wuftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 5c1cbb58..de98d02d 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -18,7 +18,7 @@ _daemon = wu-ftpd # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = ^%(__prefix_line)sfailed login from \S+ \[\]$ +failregex = ^%(__prefix_line)sfailed login from \S+ \[\]\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored.