diff --git a/ChangeLog b/ChangeLog index 14e8b2ef..edb30055 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,24 +7,69 @@ Fail2Ban (version 0.8.10) 2013/06/12 ================================================================================ -ver. 0.8.11 (2013/XX/XXX) - wanna-be-released +ver. 0.8.11 (2013/XX/XXX) - loves-unittests ----------- - Fixes: Daniel Black & Marcel Dopita * filter.d/apache-auth -- fixed and apache auth samples provide. closes #286 + Yaroslav Halchenko + * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267 + * filter.d/apache-common.conf -- support apache 2.4 more detailed error + log format. Closes gh-268 + * Backends changes detection and parsing. Close gh-223 and gh-103: + - Polling backend: detect changes in the files not only based on + mtime, but also on the size and inode. It should allow for + better detection of changes and log rotations on busy servers, + older python 2.4, and file systems with precision of mtime only + up to a second (e.g. ext3). + - All backends, possible race condition: do not read from a file + initially reported empty. Originally could have lead to + accounting for detected log lines multiple times. + Daniel Black & Мернов Георгий + * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in , + Daniel Black + * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across + all platforms to ensure permissions are the same before and after a ban - + closes gh-266 - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added + Daniel Black + * filter.d/exim-spam.conf -- a splitout of exim's spam regexes + with additions for greater control over filtering spam. - Enhancements: Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening and extra failure examples in sample logs * filter.d/apache-auth - added expressions for mod_authz, mod_auth and mod_auth_digest failures. - Daniel Black & Georgiy Mernov + Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs + Daniel Black & Sebastian Arcus + * filter.d/asterisk -- more regexes + Yaroslav Halchenko + * fail2ban-regex -- refactored to provide more details (missing and + ignored lines, control over logging, etc) while maintaining look&feel + * fail2ban-client -- log to standard error. Closes gh-264 + * Fail to configure if not a single log file was found for an + enabled jail. Closes gh-63 + * is now enforced to end with an alphanumeric + * filter.d/roundcube-auth.conf -- anchored version + Alexander Dietrich + * action.d/sendmail-common.conf -- added common sendmail settings file + and made the sender display name configurable + Steven Hiscocks + * filter.d/dovecot - Addition of session, time values and possible blank + user + * filter.d/{courier{login,smtp},proftpd,sieve,wuftpd,xinetd} - General + regex impovements + * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which + will require changing in jail.{conf,local} if using this filter. + Zurd + * filter.d/postfix - add filter for VRFY failures. closes gh-322. + ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/DEVELOP b/DEVELOP index 2d7a043b..61248393 100644 --- a/DEVELOP +++ b/DEVELOP @@ -39,8 +39,9 @@ Filters * Include sample logs with 1.2.3.4 used for IP addresses and example.com/example.org used for DNS names -* Ensure ./fail2ban-regex testcases/files/logs/{samplelog} config/filter.d/{filter}.conf - has matches for EVERY regex +* Ensure sample log is provided in testcases/files/logs/ with same name as the + filter. Each log line should include match meta data for time & IP above + every line (see other sample log files for examples) * Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if \d+ is sufficient * Use the functionality of regexs http://docs.python.org/2/library/re.html @@ -252,6 +253,10 @@ Use the text "closes #333"/"resolves #333 "/"fixes #333" where 333 represents an issue that is closed. Other text and details in link below. See: https://help.github.com/articles/closing-issues-via-commit-messages +If merge resulted in conflicts, clarify what changes were done to +corresponding files in the 'Conflicts:' section of the merge commit +message. See e.g. https://github.com/fail2ban/fail2ban/commit/f5a8a8ac + Adding Actions -------------- diff --git a/MANIFEST b/MANIFEST index 5491c7d5..f637dca0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -101,7 +101,7 @@ config/filter.d/couriersmtp.conf config/filter.d/cyrus-imap.conf config/filter.d/exim.conf config/filter.d/gssftpd.conf -config/filter.d/lighttpd-fastcgi.conf +config/filter.d/suhosin.conf config/filter.d/named-refused.conf config/filter.d/postfix.conf config/filter.d/proftpd.conf diff --git a/README.Solaris b/README.Solaris index 10a5f88c..86f56241 100644 --- a/README.Solaris +++ b/README.Solaris @@ -69,27 +69,10 @@ FAIL2BAN CONFIGURATION OPT: Create /etc/fail2ban/fail2ban.local containing: -# Fail2Ban main configuration file -# -# Comments: use '#' for comment lines and ';' (following a space) for inline comments -# -# Changes: in most of the cases you should not modify this -# file, but provide customizations in fail2ban.local file, e.g.: -# -# [Definition] -# loglevel = 4 +# Fail2Ban configuration file for logging fail2ban on Solaris # [Definition] -# Option: logtarget -# Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. -# Only one log target can be specified. -# If you change logtarget from the default value and you are -# using logrotate -- also adjust or disable rotation in the -# corresponding configuration file -# (e.g. /etc/logrotate.d/fail2ban on Debian systems) -# Values: STDOUT STDERR SYSLOG file Default: /var/log/fail2ban.log -# logtarget = /var/adm/fail2ban.log @@ -105,7 +88,7 @@ ignoreregex = for myuser from logpath = /var/adm/auth.log Set the sendmail dest address to something useful or drop the line to stop it spamming you. -Set 'myuser' to your username to avoid banning yourself or drop it. +Set 'myuser' to your username to avoid banning yourself or remove the line. START (OR RESTART) FAIL2BAN @@ -128,7 +111,7 @@ GOTCHAS AND FIXMES svcadm enable fail2ban * If svcs -xv says that fail2ban failed to start or svcs says it's in maintenance mode - chcek /var/svc/log/network-fail2ban:default.log for clues. + check /var/svc/log/network-fail2ban:default.log for clues. Check permissions on /var/adm, /var/adm/auth.log /var/adm/fail2ban.log and /var/run/fail2ban You may need to: diff --git a/THANKS b/THANKS index 26e8041f..84403239 100644 --- a/THANKS +++ b/THANKS @@ -18,6 +18,7 @@ Daniel Black David Nutter Eric Gerbier Enrico Labedzki +ftoppi Georgiy Mernov Guillaume Delvit Hanno 'Rince' Wagner @@ -33,6 +34,7 @@ Mark Edgington Markus Hoffmann Marvin Rouge mEDI +Мернов Георгий Michael C. Haller Michael Hanselmann NickMunger @@ -41,6 +43,7 @@ Raphaël Marichez René Berber Robert Edeker Russell Odom +Sebastian Arcus Sireyessire silviogarbes Stephen Gildea diff --git a/client/jailreader.py b/client/jailreader.py index f8757e26..1a2be432 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -103,16 +103,30 @@ class JailReader(ConfigReader): logSys.warn("No actions were defined for %s" % self.__name) return True - def convert(self): + def convert(self, allow_no_files=False): + """Convert read before __opts to the commands stream + + Parameters + ---------- + allow_missing : bool + Either to allow log files to be missing entirely. Primarily is + used for testing + """ + stream = [] for opt in self.__opts: if opt == "logpath": + found_files = 0 for path in self.__opts[opt].split("\n"): pathList = glob.glob(path) if len(pathList) == 0: - logSys.error("No file found for " + path) + logSys.error("No file(s) found for glob %s" % path) for p in pathList: + found_files += 1 stream.append(["set", self.__name, "addlogpath", p]) + if not (found_files or allow_no_files): + raise ValueError( + "Have not found any log file for %s jail" % self.__name) elif opt == "backend": backend = self.__opts[opt] elif opt == "maxretry": diff --git a/client/jailsreader.py b/client/jailsreader.py index 098b525d..408a40bf 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -79,14 +79,23 @@ class JailsReader(ConfigReader): return False return True - def convert(self): + def convert(self, allow_no_files=False): + """Convert read before __opts and jails to the commands stream + + Parameters + ---------- + allow_missing : bool + Either to allow log files to be missing entirely. Primarily is + used for testing + """ + stream = list() for opt in self.__opts: if opt == "": stream.append([]) # Convert jails for jail in self.__jails: - stream.extend(jail.convert()) + stream.extend(jail.convert(allow_no_files=allow_no_files)) # Start jails for jail in self.__jails: stream.append(["start", jail.getName()]) diff --git a/common/__init__.py b/common/__init__.py index 2b76f4b6..3eae8ee3 100644 --- a/common/__init__.py +++ b/common/__init__.py @@ -23,3 +23,8 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" + +import logging + +# Custom debug level +logging.HEAVYDEBUG = 5 diff --git a/config/action.d/hostsdeny.conf b/config/action.d/hostsdeny.conf index 50a4545c..36e34948 100644 --- a/config/action.d/hostsdeny.conf +++ b/config/action.d/hostsdeny.conf @@ -39,7 +39,7 @@ actionban = IP= && # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = IP= && sed /ALL:\ $IP/d > .new && mv .new +actionunban = echo "/ALL: $/
d
w
q" | ed [Init] diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf index bec1e91c..f5ca6c10 100644 --- a/config/action.d/sendmail-buffered.conf +++ b/config/action.d/sendmail-buffered.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -11,7 +15,7 @@ # Values: CMD # actionstart = printf %%b "Subject: [Fail2Ban] : started - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = if [ -f ]; then printf %%b "Subject: [Fail2Ban] : summary - From: Fail2Ban <> + From: <> To: \n Hi,\n These hosts have been banned by Fail2Ban.\n @@ -58,7 +62,7 @@ actionban = printf %%b "`date`: ( failures)\n" >> LINE=$( wc -l | awk '{ print $1 }' ) if [ $LINE -ge ]; then printf %%b "Subject: [Fail2Ban] : summary - From: Fail2Ban <> + From: <> To: \n Hi,\n These hosts have been banned by Fail2Ban.\n @@ -82,14 +86,6 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - # Default number of lines that are buffered # lines = 5 diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf new file mode 100644 index 00000000..e2820470 --- /dev/null +++ b/config/action.d/sendmail-common.conf @@ -0,0 +1,23 @@ +# Fail2Ban configuration file +# +# Common settings for sendmail actions +# +# Users can override the defaults in sendmail-common.local + +[INCLUDES] + +after = sendmail-common.local + +[Init] + +# Recipient mail address +# +dest = root + +# Sender mail address +# +sender = fail2ban + +# Sender display name +# +sendername = Fail2Ban diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index bc5074c6..2cb27bd2 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -72,14 +76,6 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - # Path to the log files which contain relevant lines for the abuser IP # logpath = /dev/null diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index 0d1fd97e..b111e19f 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -70,11 +74,3 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index 8054050d..55d388fc 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = sendmail-common.conf + [Definition] # Option: actionstart @@ -12,7 +16,7 @@ # actionstart = printf %%b "Subject: [Fail2Ban] : started Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been started successfully.\n @@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started # actionstop = printf %%b "Subject: [Fail2Ban] : stopped Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The jail has been stopped.\n @@ -46,7 +50,7 @@ actioncheck = # actionban = printf %%b "Subject: [Fail2Ban] : banned Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"` - From: Fail2Ban <> + From: <> To: \n Hi,\n The IP has just been banned by Fail2Ban after @@ -68,11 +72,3 @@ actionunban = # name = default -# Destination/Addressee of the mail -# -dest = root - -# Sender of the mail -# -sender = fail2ban - diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index ee9c84a0..cc35ae5f 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -14,4 +14,8 @@ after = apache-common.local [DEFAULT] # Common prefix for [error] apache messages which also would include -_apache_error_client = \[[^]]+\] \[(\w+:)?error\] (\[pid \d+:tid \d+\] )?\[client (:\d+)?\] +# Depending on the version it could be +# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] +# 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652] +# Reference: https://github.com/fail2ban/fail2ban/issues/268 +_apache_error_client = \[[^]]+\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\]( \S+:)? diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index a8f65e09..e4e5de5f 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -17,19 +17,22 @@ before = common.conf # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ - ^\[\]\s*NOTICE%(__pid_re)s\[\S+\] \S+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed to authenticate as '[^']*'$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: No registration for peer '[^']*' \(from \)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ - ^\[\]\s*NOTICE%(__pid_re)s \S+: Failed to authenticate user [^@]+@\S*$ - ^\[\]\s*SECURITY%(__pid_re)s \S+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ +log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d* + +failregex = ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ + ^%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ + ^%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^%(log_prefix)s Host failed to authenticate as '[^']*'$ + ^%(log_prefix)s No registration for peer '[^']*' \(from \)$ + ^%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^%(log_prefix)s Failed to authenticate user [^@]+@\S*$ + ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*>;tag=\w+\S*$ + ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"(,Challenge="\w+",ReceivedChallenge="\w+")?(,ReceivedHash="[\da-f]+")?$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index aafeb5ca..d44a6325 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -33,7 +33,7 @@ __daemon_extra_re = (?:\[ID \d+ \S+\]) # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] -__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:) +__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?) # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] diff --git a/config/filter.d/courierlogin.conf b/config/filter.d/courierlogin.conf index 20731e5d..f096325e 100644 --- a/config/filter.d/courierlogin.conf +++ b/config/filter.d/courierlogin.conf @@ -5,8 +5,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -14,7 +23,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = LOGIN FAILED, .*, ip=\[\]$ +failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[\]$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/couriersmtp.conf b/config/filter.d/couriersmtp.conf index 6c0cf5ff..65ffa5d7 100644 --- a/config/filter.d/couriersmtp.conf +++ b/config/filter.d/couriersmtp.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = courieresmtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,7 +22,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = error,relay=,.*550 User unknown +failregex = ^%(__prefix_line)serror,relay=,.*: 550 User unknown\.$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index 758f75de..0ace92c1 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = (?:cyrus/)?(?:imapd?|pop3d?) + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,10 +22,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = : badlogin: .*\[\] plaintext .*SASL\(-13\): authentication failure: checkpass failed$ - : badlogin: .*\[\] LOGIN \[SASL\(-13\): authentication failure: checkpass failed\]$ - : badlogin: .*\[\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$ - : badlogin: .*\[\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ .*?\[?SASL\(-13\): authentication failure: .*\]?$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index dd4c35ba..2143e224 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -16,8 +16,8 @@ _daemon = dovecot(-auth)? # first regex is essentially a copy of pam-generic.conf # Values: TEXT # -failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3},( TLS( handshaking)?(: Disconnected)?)?\s*$ +failregex = ^%(__prefix_line)s(pam_unix(\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use disabled \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf new file mode 100644 index 00000000..79d6ffbb --- /dev/null +++ b/config/filter.d/exim-common.conf @@ -0,0 +1,17 @@ +# Fail2Ban configuration file for exim +# +# Author: Daniel Black +# + +[INCLUDES] + +# Load customizations if any available +# +after = exim-common.local + +[Definition] + +# From exim source code: ./src/receive.c:add_host_info_for_log +host_info = H=([\w.-]+ )?(\(\S+\) )?\[\](:\d+)? (I=\[\S+\]:\d+ )?(U=\S+ )?(P=e?smtp )? +pid = ( \[\d+\])? + diff --git a/config/filter.d/exim-spam.conf b/config/filter.d/exim-spam.conf new file mode 100644 index 00000000..55a6f5dd --- /dev/null +++ b/config/filter.d/exim-spam.conf @@ -0,0 +1,29 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Daniel Black (rewrote with strong regexs) +# + + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# exim-common.local +before = exim-common.conf + + +[Definition] + +# Option: failregex +# Notes.: This includes the spam rejection messages of exim. +# Note the %(host_info) defination contains a match + +failregex = ^%(pid)s \S+ F=(<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$ + ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$ + ^%(pid)s \S+ %(host_info)sF=(<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index d335c9fc..63b0fa1d 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -1,20 +1,29 @@ # Fail2Ban configuration file # # Author: Cyril Jaquier +# Daniel Black (rewrote with strong regexs) # -# + + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# exim-common.local +before = exim-common.conf + [Definition] # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# Values: TEXT -# -failregex = ^ H=\S+ \(\S+\) \[\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$ - ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ +# Notes.: This includes the rejection messages of exim. For spam and filter +# related bans use the exim-spam.conf +# Note the %(host_info) defination contains a match + +failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ + ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ + ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f92c3619..da981733 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = postfix/smtpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -13,8 +22,9 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = reject: RCPT from (.*)\[\]: 554 - reject: RCPT from (.*)\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 13080fcc..d3120216 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -13,6 +13,8 @@ before = common.conf [Definition] +_deamon = proftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -20,10 +22,12 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ - ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ + +__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 7b153f44..fe669f66 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = common.conf + [Definition] # Option: failregex @@ -13,7 +17,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = (FAILED login|Login failed) for .* from \s*$ +failregex = ^\s*(\[(\s\+[0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/sieve.conf b/config/filter.d/sieve.conf index 866b4228..b2af6774 100644 --- a/config/filter.d/sieve.conf +++ b/config/filter.d/sieve.conf @@ -4,15 +4,24 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_deamon = (?:cyrus/)?(?:tim)?sieved? + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can # be used for standard IP/hostname matching. # Values: TEXT # -failregex = : badlogin: .*\[\] (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failure$ +failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ authentication failure$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/lighttpd-fastcgi.conf b/config/filter.d/suhosin.conf similarity index 67% rename from config/filter.d/lighttpd-fastcgi.conf rename to config/filter.d/suhosin.conf index 1c6e3fce..c79c157f 100644 --- a/config/filter.d/lighttpd-fastcgi.conf +++ b/config/filter.d/suhosin.conf @@ -9,7 +9,8 @@ # Notes.: regex to match ALERTS as notified by lighttpd's FastCGI Module # Values: TEXT # -failregex = .*ALERT\ -\ .*attacker\ \'\' +# https://github.com/stefanesser/suhosin/blob/1fba865ab73cc98a3109f88d85eb82c1bfc29b37/log.c#L161 +failregex = ALERT - .* \(attacker '', file '.*'(?:, line \d+)?\)$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/wuftpd.conf b/config/filter.d/wuftpd.conf index 3351d258..de98d02d 100644 --- a/config/filter.d/wuftpd.conf +++ b/config/filter.d/wuftpd.conf @@ -4,14 +4,21 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] +_daemon = wu-ftpd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = wu-ftpd(?:\[\d+\])?:\s+\(pam_unix\)\s+authentication failure.* rhost=$ - wu-ftpd(?:\[\d+\])?: *failed login from .*\[\] *$ +failregex = ^%(__prefix_line)sfailed login from \S+ \[\]\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/xinetd-fail.conf b/config/filter.d/xinetd-fail.conf index 4ff5bfde..253ce15d 100644 --- a/config/filter.d/xinetd-fail.conf +++ b/config/filter.d/xinetd-fail.conf @@ -4,8 +4,17 @@ # # +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + [Definition] +_daemon = xinetd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -19,8 +28,8 @@ # load => xinetd: max_load (temporary problem) # -failregex = xinetd(?:\[\d{1,5}\])?: FAIL: \S+ address from=$ - xinetd(?:\[\d{1,5}\])?: FAIL: \S+ libwrap from=$ +failregex = ^%(__prefix_line)sFAIL: \S+ address from=$ + ^%(__prefix_line)sFAIL: \S+ libwrap from=$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/jail.conf b/config/jail.conf index c999cc7b..640a5874 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -65,7 +65,7 @@ usedns = warn enabled = false filter = sshd action = iptables[name=SSH, port=ssh, protocol=tcp] - sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com] + sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] logpath = /var/log/sshd.log maxretry = 5 @@ -229,7 +229,7 @@ logpath = /var/log/apache2/error_log enabled = false filter = roundcube-auth -action = iptables[name=RoundCube, port="http,https"] +action = iptables-multiport[name=RoundCube, port="http,https"] logpath = /var/log/roundcube/userlogins @@ -241,7 +241,7 @@ enabled = false filter = sogo-auth # without proxy this would be: # port = 20000 -action = iptables[name=SOGo, port="http,https"] +action = iptables-multiport[name=SOGo, port="http,https"] logpath = /var/log/sogo/sogo.log # Ban attackers that try to use PHP's URL-fopen() functionality @@ -251,35 +251,25 @@ logpath = /var/log/sogo/sogo.log [php-url-fopen] enabled = false -action = iptables[name=php-url-open, port="http,https"] +action = iptables-multiport[name=php-url-open, port="http,https"] filter = php-url-fopen logpath = /var/www/*/logs/access_log maxretry = 1 -# A simple PHP-fastcgi jail which works with lighttpd. -# If you run a lighttpd server, then you probably will -# find these kinds of messages in your error_log: -# ALERT – tried to register forbidden variable ‘GLOBALS’ -# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') -# This jail would block the IP 1.2.3.4. - -[lighttpd-fastcgi] +[suhosin] enabled = false -filter = lighttpd-fastcgi -action = iptables[name=lighttpd-fastcgi, port="http,https"] +filter = suhosin +action = iptables-multiport[name=suhosin, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 -# Same as above for mod_auth -# It catches wrong authentications - [lighttpd-auth] enabled = false filter = lighttpd-auth -action = iptables[name=lighttpd-auth, port="http,https"] +action = iptables-multiport[name=lighttpd-auth, port="http,https"] # adapt the following two items as needed logpath = /var/log/lighttpd/error.log maxretry = 2 @@ -405,6 +395,17 @@ maxretry=5 enabled = false filter = 3proxy -action = iptables-multiport[name=3proxy, port=3128, protocol=tcp] +action = iptables[name=3proxy, port=3128, protocol=tcp] logpath = /var/log/3proxy.log +[exim] +enabled = false +filter = exim +action = iptables-multiport[name=exim,port="25,465,587"] +logpath = /var/log/exim/mainlog + +[exim-spam] +enabled = false +filter = exim-spam +action = iptables-multiport[name=exim-spam,port="25,465,587"] +logpath = /var/log/exim/mainlog diff --git a/fail2ban-client b/fail2ban-client index de8519f4..be48a613 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -229,7 +229,7 @@ class Fail2banClient: elif len(cmd) == 2 and cmd[0] == "reload": if self.__ping(): jail = cmd[1] - ret = self.__readJailConfig(jail) + ret = self.__readConfig(jail) # Do not continue if configuration is not 100% valid if not ret: return False @@ -336,13 +336,13 @@ class Fail2banClient: logSys.setLevel(logging.INFO) else: logSys.setLevel(logging.DEBUG) - # Add the default logging handler - stdout = logging.StreamHandler(sys.stdout) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) # set a format which is simpler for console use formatter = logging.Formatter('%(levelname)-6s %(message)s') # tell the handler to use this format - stdout.setFormatter(formatter) - logSys.addHandler(stdout) + logout.setFormatter(formatter) + logSys.addHandler(logout) # Set the configuration path self.__configurator.setBaseDir(self.__conf["conf"]) @@ -394,19 +394,18 @@ class Fail2banClient: return False return self.__processCommand(args) - def __readConfig(self): + def __readConfig(self, jail=None): # Read the configuration - self.__configurator.readAll() - ret = self.__configurator.getOptions() - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() - return ret - - def __readJailConfig(self, jail): - self.__configurator.readAll() - ret = self.__configurator.getOptions(jail) - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() + # TODO: get away from stew of return codes and exception + # handling -- handle via exceptions + try: + self.__configurator.readAll() + ret = self.__configurator.getOptions(jail) + self.__configurator.convertToProtocol() + self.__stream = self.__configurator.getConfigStream() + except Exception, e: + logSys.error("Failed during configuration: %s" % e) + ret = False return ret #@staticmethod diff --git a/fail2ban-regex b/fail2ban-regex index a0a90b05..e19b1bc9 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -17,9 +17,17 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +Report bugs to https://github.com/fail2ban/fail2ban/issues +""" __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" import getopt, sys, time, logging, os @@ -32,231 +40,238 @@ except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") from common.version import version +from optparse import OptionParser, Option + from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException +from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.regex") +logSys = logging.getLogger("fail2ban") -class RegexStat: +def shortstr(s, l=53): + """Return shortened string + """ + if len(s) > l: + return s[:l-3] + '...' + return s + +def pprint_list(l, header=None): + if not len(l): + return + if header: + s = "|- %s\n" % header + else: + s = '' + print s + "| " + "\n| ".join(l) + '\n`-' + +def get_opt_parser(): + # use module docstring for help output + p = OptionParser( + usage="%s [OPTIONS] [IGNOREREGEX]\n" % sys.argv[0] + __doc__ + + """ +LOG: + string a string representing a log line + filename path to a log file (/var/log/auth.log) + +REGEX: + string a string representing a 'failregex' + filename path to a filter file (filter.d/sshd.conf) + +IGNOREREGEX: + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +""", + version="%prog " + version) + + p.add_options([ + Option('-l', "--log-level", type="choice", + dest="log_level", + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), + default=None, + help="Log level for the Fail2Ban logger to use"), + Option("-v", "--verbose", action='store_true', + help="Be verbose in output"), + Option("--print-all-missed", action='store_true', + help="Either to print all missed lines"), + Option("--print-all-ignored", action='store_true', + help="Either to print all ignored lines"), + Option("-t", "--log-traceback", action='store_true', + help="Enrich log-messages with compressed tracebacks"), + Option("--full-traceback", action='store_true', + help="Either to make the tracebacks full, not compressed (as by default)"), + + ]) + + return p + + +class RegexStat(object): def __init__(self, failregex): - self.__stats = 0 - self.__failregex = failregex - self.__ipList = list() + self._stats = 0 + self._failregex = failregex + self._ipList = list() def __str__(self): return "%s(%r) %d failed: %s" \ - % (self.__class__, self.__failregex, self.__stats, self.__ipList) + % (self.__class__, self._failregex, self._stats, self._ipList) def inc(self): - self.__stats += 1 + self._stats += 1 def getStats(self): - return self.__stats + return self._stats def getFailRegex(self): - return self.__failregex + return self._failregex def appendIP(self, value): - self.__ipList.extend(value) + self._ipList.append(value) def getIPList(self): - return self.__ipList + return self._ipList -class Fail2banRegex: - test = None +class LineStats(object): + """Just a convenience container for stats + """ + def __init__(self): + self.tested = self.matched = 0 + self.missed_lines = [] + self.ignored_lines = [] + + def __str__(self): + return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self + + @property + def ignored(self): + return len(self.ignored_lines) + + @property + def missed(self): + return self.tested - (self.ignored + self.matched) + + # just for convenient str + def __getitem__(self, key): + return getattr(self, key) + + +class Fail2banRegex(object): CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} - def __init__(self): - self.__filter = Filter(None) - self.__ignoreregex = list() - self.__failregex = list() - self.__verbose = False - # Setup logging - logging.getLogger("fail2ban").handlers = [] - self.__hdlr = logging.StreamHandler(Fail2banRegex.test) - # set a format which is simpler for console use - formatter = logging.Formatter("%(message)s") - # tell the handler to use this format - self.__hdlr.setFormatter(formatter) - self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self.__hdlr) - logging.getLogger("fail2ban").setLevel(logging.ERROR) + def __init__(self, opts): + self._verbose = opts.verbose + self._print_all_missed = opts.print_all_missed + self._print_all_ignored = opts.print_all_ignored - #@staticmethod - def dispVersion(): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - dispVersion = staticmethod(dispVersion) + self._filter = Filter(None) + self._ignoreregex = list() + self._failregex = list() + self._line_stats = LineStats() - #@staticmethod - def dispUsage(): - print "Usage: "+sys.argv[0]+" [OPTIONS] [IGNOREREGEX]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "This tools can test regular expressions for \"fail2ban\"." - print - print "Options:" - print " -h, --help display this help message" - print " -V, --version print the version" - print " -v, --verbose verbose output" - print - print "Log:" - print " string a string representing a log line" - print " filename path to a log file (/var/log/auth.log)" - print - print "Regex:" - print " string a string representing a 'failregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "IgnoreRegex:" - print " string a string representing an 'ignoreregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" - dispUsage = staticmethod(dispUsage) - def getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) - elif opt[0] in ["-v", "--verbose"]: - self.__verbose = True - - #@staticmethod - def logIsFile(value): - return os.path.isfile(value) - logIsFile = staticmethod(logIsFile) - - def readIgnoreRegex(self, value): + def readRegex(self, value, regextype): + assert(regextype in ('fail', 'ignore')) + regex = regextype + 'regex' if os.path.isfile(value): reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) try: reader.read(value) - print "Use ignoreregex file : " + value - self.__ignoreregex = [RegexStat(m) - for m in reader.get("Definition", "ignoreregex").split('\n')] + print "Use %11s file : %s" % (regex, value) + # TODO: reuse functionality in client + regex_values = [ + RegexStat(m) + for m in reader.get("Definition", regex).split('\n') + if m != ""] except NoSectionError: - print "No [Definition] section in " + value - print + print "No [Definition] section in %s" % value return False except NoOptionError: - print "No failregex option in " + value - print + print "No %s option in %s" % (regex, value) return False except MissingSectionHeaderError: - print "No section headers in " + value - print + print "No section headers in %s" % value return False else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use ignoreregex line : " + stripReg - self.__ignoreregex = [RegexStat(value)] - return True + print "Use %11s line : %s" % (regex, shortstr(value)) + regex_values = [RegexStat(value)] - def readRegex(self, value): - if os.path.isfile(value): - reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) - try: - reader.read(value) - print "Use regex file : " + value - self.__failregex = [RegexStat(m) - for m in reader.get("Definition", "failregex").split('\n')] - except NoSectionError: - print "No [Definition] section in " + value - print - return False - except NoOptionError: - print "No failregex option in " + value - print - return False - except MissingSectionHeaderError: - print "No section headers in " + value - print - return False - else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use regex line : " + stripReg - self.__failregex = [RegexStat(value)] + setattr(self, "_" + regex, regex_values) + for regex in regex_values: + getattr( + self._filter, + 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True def testIgnoreRegex(self, line): found = False - for regex in self.__ignoreregex: - logging.getLogger("fail2ban").setLevel(self.__logging_level) - try: - self.__filter.addIgnoreRegex(regex.getFailRegex()) - try: - ret = self.__filter.ignoreLine(line) - if ret: - regex.inc() - except RegexException, e: - print e - return False - finally: - self.__filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self.__logging_level) + try: + ret = self._filter.ignoreLine(line) + if ret is not None: + found = True + regex = self._ignoreregex[ret].inc() + except RegexException, e: + print e + return False + return found def testRegex(self, line): - found = False - for regex in self.__ignoreregex: - self.__filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self.__failregex: - logging.getLogger("fail2ban").setLevel(logging.DEBUG) - try: - self.__filter.addFailRegex(regex.getFailRegex()) - try: - ret = self.__filter.processLine(line) - if not len(ret) == 0: - if found == True: - ret[0].append(True) - else: - found = True - ret[0].append(False) - regex.inc() - regex.appendIP(ret) - except RegexException, e: - print e - return False - except IndexError: - print "Sorry, but no found in regex" - return False - finally: - self.__filter.delFailRegex(0) - logging.getLogger("fail2ban").setLevel(logging.CRITICAL) - for regex in self.__ignoreregex: - self.__filter.delIgnoreRegex(0) + try: + ret = self._filter.processLine(line, checkAllRegex=True) + for match in ret: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) + except RegexException, e: + print e + return False + except IndexError: + print "Sorry, but no found in regex" + return False + return len(ret) > 0 + + + def process(self, test_lines): + + for line in test_lines: + if line.startswith('#') or not line.strip(): + # skip comment and empty lines + continue + is_ignored = fail2banRegex.testIgnoreRegex(line) + if is_ignored: + self._line_stats.ignored_lines.append(line) + + if fail2banRegex.testRegex(line): + assert(not is_ignored) + self._line_stats.matched += 1 + else: + if not is_ignored: + self._line_stats.missed_lines.append(line) + self._line_stats.tested += 1 + + def printLines(self, ltype): + lstats = self._line_stats + assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) + l = lstats[ltype + '_lines'] + if len(l): + header = "%s line(s):" % (ltype.capitalize(),) + if len(l) < 20 or getattr(self, '_print_all_' + ltype): + pprint_list([x.rstrip() for x in l], header) + else: + print "%s: too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, len(l)) def printStats(self): print print "Results" print "=======" - print def print_failregexes(title, failregexes): # Print title @@ -264,106 +279,110 @@ class Fail2banRegex: for cnt, failregex in enumerate(failregexes): match = failregex.getStats() total += match - if (match or self.__verbose): - out.append("| %d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - print "%s: %d total" % (title, total) - if len(out): - print "|- #) [# of hits] regular expression" - print '\n'.join(out) - print '`-' - print + if (match or self._verbose): + out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) + + if self._verbose and len(failregex.getIPList()): + for ip in failregex.getIPList(): + timeTuple = time.localtime(ip[2]) + timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) + out.append( + " %s %s%s" % ( + ip[1], + timeString, + ip[3] and " (multiple regex matched)" or "")) + + print "\n%s: %d total" % (title, total) + pprint_list(out, " #) [# of hits] regular expression") return total # Print title - total = print_failregexes("Failregex", self.__failregex) - _ = print_failregexes("Ignoreregex", self.__ignoreregex) + total = print_failregexes("Failregex", self._failregex) + _ = print_failregexes("Ignoreregex", self._ignoreregex) - print "Summary" - print "=======" - print - if total == 0: - print "Sorry, no match" - print - print "Look at the above section 'Running tests' which could contain important" - print "information." - return False - else: - # Print stats - print "Addresses found:" - for cnt, failregex in enumerate(self.__failregex): - if self.__verbose or len(failregex.getIPList()): - print "[%d]" % (cnt+1) - for ip in failregex.getIPList(): - timeTuple = time.localtime(ip[1]) - timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - print " %s (%s)%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "") - print + print "\nDate template hits:" + out = [] + for template in self._filter.dateDetector.getTemplates(): + if self._verbose or template.getHits(): + out.append("[%d] %s" % (template.getHits(), template.getName())) + pprint_list(out, "[# of hits] date format") - print "Date template hits:" - for template in self.__filter.dateDetector.getTemplates(): - if self.__verbose or template.getHits(): - print `template.getHits()` + " hit(s): " + template.getName() - print + print "\nLines: %s" % self._line_stats - print "Success, the total number of match is " + str(total) - print - print "However, look at the above section 'Running tests' which could contain important" - print "information." - return True + self.printLines('ignored') + self.printLines('missed') + + return True if __name__ == "__main__": - fail2banRegex = Fail2banRegex() - # Reads the command line options. - try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] - optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - fail2banRegex.dispUsage() - sys.exit(-1) - # Process command line - fail2banRegex.getCmdLineOptions(optList) + + parser = get_opt_parser() + (opts, args) = parser.parse_args() + + fail2banRegex = Fail2banRegex(opts) # We need 2 or 3 parameters if not len(args) in (2, 3): - fail2banRegex.dispUsage() + sys.stderr.write("ERROR: provide both and .\n\n") + parser.print_help() sys.exit(-1) + + # TODO: taken from -testcases -- move common functionality somewhere + if opts.log_level is not None: # pragma: no cover + # so we had explicit settings + logSys.setLevel(getattr(logging, opts.log_level.upper())) + else: # pragma: no cover + # suppress the logging but it would leave unittests' progress dots + # ticking, unless like with '-l fatal' which would be silent + # unless error occurs + logSys.setLevel(getattr(logging, 'FATAL')) + + # Add the default logging handler + stdout = logging.StreamHandler(sys.stdout) + + fmt = 'D: %(message)s' + + if opts.log_traceback: + Formatter = FormatterWithTraceBack + fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt else: - print - print "Running tests" - print "=============" - print + Formatter = logging.Formatter - cmd_log, cmd_regex = args[:2] + # Custom log format for the verbose tests runs + if opts.verbose > 1: # pragma: no cover + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) + else: # pragma: no cover + # just prefix with the space + stdout.setFormatter(Formatter(fmt)) + logSys.addHandler(stdout) - if len(args) == 3: - fail2banRegex.readIgnoreRegex(args[2]) or sys.exit(-1) + print + print "Running tests" + print "=============" + print - fail2banRegex.readRegex(cmd_regex) or sys.exit(-1) + cmd_log, cmd_regex = args[:2] - if fail2banRegex.logIsFile(cmd_log): - try: - hdlr = open(cmd_log) - print "Use log file : " + cmd_log - print - for line in hdlr: - fail2banRegex.testIgnoreRegex(line) - fail2banRegex.testRegex(line) - except IOError, e: - print e - print - sys.exit(-1) - else: - if len(sys.argv[1]) > 53: - stripLog = cmd_log[0:50] + "..." - else: - stripLog = cmd_log - print "Use single line: " + stripLog - print - fail2banRegex.testIgnoreRegex(cmd_log) - fail2banRegex.testRegex(cmd_log) + if len(args) == 3: + fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log) + print "Use log file : %s" % cmd_log + test_lines = hdlr.readlines() + except IOError, e: + print e + sys.exit(-1) + else: + print "Use single line : %s" % shortstr(cmd_log) + test_lines = [ cmd_log ] + print + + fail2banRegex.process(test_lines) + + fail2banRegex.printStats() or sys.exit(-1) diff --git a/fail2ban-testcases b/fail2ban-testcases index 0bd70259..5aed6e43 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -27,6 +27,14 @@ __license__ = "GPL" import unittest, logging, sys, time, os +if sys.version_info >= (2, 6): + import json +else: + try: + import simplejson as json + except ImportError: + json = None + from common.version import version from testcases import banmanagertestcase from testcases import clientreadertestcase @@ -37,6 +45,8 @@ from testcases import datedetectortestcase from testcases import actiontestcase from testcases import sockettestcase from testcases import misctestcase +if json: + from testcases import samplestestcase from testcases.utils import FormatterWithTraceBack from server.mytime import MyTime @@ -52,7 +62,7 @@ def get_opt_parser(): p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), default=None, help="Log level for the logger to use during running tests"), Option('-n', "--no-network", action="store_true", @@ -76,9 +86,10 @@ parser = get_opt_parser() logSys = logging.getLogger("fail2ban") # Numerical level of verbosity corresponding to a log "level" -verbosity = {'debug': 3, +verbosity = {'heavydebug': 4, + 'debug': 3, 'info': 2, - 'warn': 1, + 'warning': 1, 'error': 1, 'fatal': 0, None: 1}[opts.log_level] @@ -169,6 +180,11 @@ tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) +if json: + # Filter Regex tests with sample logs + tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) +else: + print "I: Skipping filter samples testing. No simplejson/json module" # # Extensive use-tests of different available filters backends diff --git a/server/datedetector.py b/server/datedetector.py index 0c8b4df2..0ed9e00a 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -174,6 +174,7 @@ class DateDetector: match = template.matchDate(line) if not match is None: logSys.debug("Matched time template %s" % template.getName()) + template.incHits() return match return None finally: diff --git a/server/datetemplate.py b/server/datetemplate.py index 8c49aa15..86eeee8e 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -59,11 +59,12 @@ class DateTemplate: def getHits(self): return self.__hits + + def incHits(self): + self.__hits += 1 def matchDate(self, line): dateMatch = self.__cRegex.search(line) - if not dateMatch is None: - self.__hits += 1 return dateMatch def getDate(self, line): diff --git a/server/failregex.py b/server/failregex.py index 957c77db..bfa50575 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -41,7 +41,7 @@ class Regex: self._matchCache = None # Perform shortcuts expansions. # Replace "" with default regular expression for host. - regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]+)") + regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]*\w)") if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: @@ -50,7 +50,8 @@ class Regex: except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - + def __str__(self): + return "%s(%r)" % (self.__class__.__name__, self._regex) ## # Gets the regular expression. # diff --git a/server/filter.py b/server/filter.py index 90530f92..2f88cbea 100644 --- a/server/filter.py +++ b/server/filter.py @@ -21,6 +21,8 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" +import sys + from failmanager import FailManagerEmpty from failmanager import FailManager from ticket import FailTicket @@ -282,7 +284,7 @@ class Filter(JailThread): return False - def processLine(self, line): + def processLine(self, line, returnRawHost=False, checkAllRegex=False): """Split the time portion from log msg and return findFailures on them """ try: @@ -291,6 +293,8 @@ class Filter(JailThread): except UnicodeDecodeError: l = line l = l.rstrip('\r\n') + + logSys.log(7, "Working on line %r", l) timeMatch = self.dateDetector.matchTime(l) if timeMatch: # Lets split into time part and log part of the line @@ -302,14 +306,15 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine) + return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager """ for element in self.processLine(line): - ip = element[0] - unixTime = element[1] + failregex = element[0] + ip = element[1] + unixTime = element[2] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): @@ -320,6 +325,7 @@ class Filter(JailThread): logSys.debug("Ignore %s" % ip) continue logSys.debug("Found %s" % ip) + ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) self.failManager.addFailure(FailTicket(ip, unixTime, [line])) ## @@ -330,11 +336,11 @@ class Filter(JailThread): # @return: a boolean def ignoreLine(self, line): - for ignoreRegex in self.__ignoreRegex: + for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): ignoreRegex.search(line) if ignoreRegex.hasMatched(): - return True - return False + return ignoreRegexIndex + return None ## # Finds the failure in a line given split into time and log parts. @@ -343,18 +349,22 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, timeLine, logLine): + def findFailure(self, timeLine, logLine, + returnRawHost=False, checkAllRegex=False): + logSys.log(5, "Date: %r, message: %r", timeLine, logLine) failList = list() # Checks if we must ignore this line. - if self.ignoreLine(logLine): + if self.ignoreLine(logLine) is not None: # The ignoreregex matched. Return. + logSys.log(7, "Matched ignoreregex and was ignored") return failList + date = self.dateDetector.getUnixTime(timeLine) # Iterates over all the regular expressions. - for failRegex in self.__failRegex: + for failRegexIndex, failRegex in enumerate(self.__failRegex): failRegex.search(logLine) if failRegex.hasMatched(): # The failregex matched. - date = self.dateDetector.getUnixTime(timeLine) + logSys.log(7, "Matched %s", failRegex) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" @@ -364,12 +374,17 @@ class Filter(JailThread): else: try: host = failRegex.getHost() - ipMatch = DNSUtils.textToIp(host, self.__useDns) - if ipMatch: - for ip in ipMatch: - failList.append([ip, date]) - # We matched a regex, it is enough to stop. - break + if returnRawHost: + failList.append([failRegexIndex, host, date]) + if not checkAllRegex: + break + else: + ipMatch = DNSUtils.textToIp(host, self.__useDns) + if ipMatch: + for ip in ipMatch: + failList.append([failRegexIndex, ip, date]) + if not checkAllRegex: + break except RegexException, e: # pragma: no cover - unsure if reachable logSys.error(e) return failList @@ -473,7 +488,7 @@ class FileFilter(Filter): return False # Try to open log file. try: - container.open() + has_content = container.open() # see http://python.org/dev/peps/pep-3151/ except IOError, e: logSys.error("Unable to open %s" % filename) @@ -488,7 +503,12 @@ class FileFilter(Filter): logSys.exception(e) return False - while True: + # yoh: has_content is just a bool, so do not expect it to + # change -- loop is exited upon break, and is not entered at + # all if upon container opening that one was empty. If we + # start reading tested to be empty container -- race condition + # might occur leading at least to tests failures. + while has_content: line = container.readline() if (line == "") or not self._isActive(): # The jail reached the bottom or has been stopped @@ -551,10 +571,20 @@ class FileContainer: fd = self.__handler.fileno() flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + # Stat the file before even attempting to read it + stats = os.fstat(self.__handler.fileno()) + if not stats.st_size: + # yoh: so it is still an empty file -- nothing should be + # read from it yet + # print "D: no content -- return" + return False firstLine = self.__handler.readline() # Computes the MD5 of the first line. myHash = md5sum(firstLine).digest() - stats = os.fstat(self.__handler.fileno()) + ## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % ( + ## self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos, + ## self.__hash != myHash or self.__ino != stats.st_ino) + ## sys.stdout.flush() # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: logSys.debug("Log rotation detected for %s" % self.__filename) @@ -563,6 +593,7 @@ class FileContainer: self.__pos = 0 # Sets the file pointer to the last position. self.__handler.seek(self.__pos) + return True def readline(self): if self.__handler is None: @@ -576,6 +607,8 @@ class FileContainer: # Closes the file. self.__handler.close() self.__handler = None + ## print "D: Closed %s with pos %d" % (handler, self.__pos) + ## sys.stdout.flush() @@ -598,9 +631,9 @@ class DNSUtils: """ try: return socket.gethostbyname_ex(dns)[2] - except socket.gaierror: - logSys.warn("Unable to find a corresponding IP address for %s" - % dns) + except socket.error, e: + logSys.warn("Unable to find a corresponding IP address for %s: %s" + % (dns, e)) return list() dnsToIp = staticmethod(dnsToIp) diff --git a/server/filterpoll.py b/server/filterpoll.py index 20cde0ca..8b4038b8 100644 --- a/server/filterpoll.py +++ b/server/filterpoll.py @@ -52,7 +52,7 @@ class FilterPoll(FileFilter): FileFilter.__init__(self, jail) self.__modified = False ## The time of the last modification of the file. - self.__lastModTime = dict() + self.__prevStats = dict() self.__file404Cnt = dict() logSys.debug("Created FilterPoll") @@ -62,7 +62,7 @@ class FilterPoll(FileFilter): # @param path log file path def _addLogPath(self, path): - self.__lastModTime[path] = 0 + self.__prevStats[path] = (0, None, None) # mtime, ino, size self.__file404Cnt[path] = 0 ## @@ -71,7 +71,7 @@ class FilterPoll(FileFilter): # @param path the log file to delete def _delLogPath(self, path): - del self.__lastModTime[path] + del self.__prevStats[path] del self.__file404Cnt[path] ## @@ -84,6 +84,9 @@ class FilterPoll(FileFilter): def run(self): self.setActive(True) while self._isActive(): + if logSys.getEffectiveLevel() <= 6: + logSys.log(6, "Woke up idle=%s with %d files monitored", + self.getIdle(), len(self.getLogPath())) if not self.getIdle(): # Get file modification for container in self.getLogPath(): @@ -118,12 +121,20 @@ class FilterPoll(FileFilter): def isModified(self, filename): try: logStats = os.stat(filename) + stats = logStats.st_mtime, logStats.st_ino, logStats.st_size + pstats = self.__prevStats[filename] self.__file404Cnt[filename] = 0 - if self.__lastModTime[filename] == logStats.st_mtime: + if logSys.getEffectiveLevel() <= 7: + # we do not want to waste time on strftime etc if not necessary + dt = logStats.st_mtime - pstats[0] + logSys.log(7, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s", + filename, pstats, stats, dt) + # os.system("stat %s | grep Modify" % filename) + if pstats == stats: return False else: - logSys.debug(filename + " has been modified") - self.__lastModTime[filename] = logStats.st_mtime + logSys.debug("%s has been modified", filename) + self.__prevStats[filename] = stats return True except OSError, e: logSys.error("Unable to get stat on %s because of: %s" diff --git a/server/jailthread.py b/server/jailthread.py index 11d1a82b..ca6effae 100644 --- a/server/jailthread.py +++ b/server/jailthread.py @@ -56,7 +56,7 @@ class JailThread(Thread): def setSleepTime(self, value): self.__sleepTime = value - logSys.info("Set sleeptime = " + value) + logSys.info("Set sleeptime %s" % value) ## # Get the time that the thread sleeps. diff --git a/testcases/actiontestcase.py b/testcases/actiontestcase.py index 9a7776ac..1aa9f24e 100644 --- a/testcases/actiontestcase.py +++ b/testcases/actiontestcase.py @@ -69,11 +69,11 @@ class ExecuteAction(unittest.TestCase): self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': ''})) self.assertFalse(Action.substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) # missing tags are ok - self.assertEquals(Action.substituteRecursiveTags({'A': ''}), {'A': ''}) - self.assertEquals(Action.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) - self.assertEquals(Action.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(Action.substituteRecursiveTags({'A': ''}), {'A': ''}) + self.assertEqual(Action.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) + self.assertEqual(Action.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # rest is just cool - self.assertEquals(Action.substituteRecursiveTags(aInfo), + self.assertEqual(Action.substituteRecursiveTags(aInfo), { 'HOST': "192.0.2.0", 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index a8c60bf8..695b3877 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -108,7 +108,7 @@ class JailReaderTest(unittest.TestCase): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] result = JailReader.splitAction(action) - self.assertEquals(expected, result) + self.assertEqual(expected, result) class JailsReaderTest(unittest.TestCase): @@ -132,7 +132,7 @@ class JailsReaderTest(unittest.TestCase): jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine - comm_commands = jails.convert() + comm_commands = jails.convert(allow_no_files=True) # by default we have lots of jails ;) self.assertTrue(len(comm_commands)) diff --git a/testcases/files/logs/3proxy b/testcases/files/logs/3proxy index 2967c9bf..614600ef 100644 --- a/testcases/files/logs/3proxy +++ b/testcases/files/logs/3proxy @@ -1,3 +1,6 @@ +# failJSON: { "time": "2013-06-11T02:09:40", "match": true , "host": "1.2.3.4" } 11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +# failJSON: { "time": "2013-06-11T02:09:43", "match": true , "host": "1.2.3.4" } 11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +# failJSON: { "time": "2013-06-13T01:39:34", "match": true , "host": "1.2.3.4" } 13-06-2013 01:39:34 +0300 PROXY.3128 00508 - 1.2.3.4:28938 0.0.0.0:0 0 0 0 diff --git a/testcases/files/logs/apache-auth b/testcases/files/logs/apache-auth index 915fabc1..7714c4bb 100644 --- a/testcases/files/logs/apache-auth +++ b/testcases/files/logs/apache-auth @@ -1,4 +1,5 @@ # Should not match -- DoS vector https://vndh.net/note:fail2ban-089-denial-service +# failJSON: { "match": false } [Sat Jun 01 02:17:42 2013] [error] [client 192.168.33.1] File does not exist: /srv/http/site/[client 192.168.0.1] user root not found # should match @@ -105,7 +106,8 @@ # failJSON: { "time": "2013-07-29T02:15:26", "match": true , "host": "127.0.0.1" } [Mon Jul 29 02:15:26 2013] [error] [client 127.0.0.1] Digest: invalid nonce LWEDr5TiBAA=ceddd011628c30e3646f7acda4f1a0ab6b7c5ae6 received - user attempted time travel - - # failJSON: { "time": "2013-07-29T02:12:55", "match": true , "host": "127.0.0.1" } [Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel + +# failJSON: { "time": "2005-06-01T02:17:42", "match": true , "host": "192.168.0.2" } +[Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found diff --git a/testcases/files/logs/apache-badbots b/testcases/files/logs/apache-badbots new file mode 100644 index 00000000..35669252 --- /dev/null +++ b/testcases/files/logs/apache-badbots @@ -0,0 +1,2 @@ +# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } +1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" diff --git a/testcases/files/logs/apache-nohome b/testcases/files/logs/apache-nohome new file mode 100644 index 00000000..aea0d816 --- /dev/null +++ b/testcases/files/logs/apache-nohome @@ -0,0 +1,6 @@ +# Apache 2.2 +# failJSON: { "time": "2005-06-01T11:23:08", "match": true , "host": "1.2.3.4" } +[Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] File does not exist: /xxx/~ +# Apache 2.4 +# failJSON: { "time": "2005-06-27T11:55:44", "match": true , "host": "192.0.2.12" } +[Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 192.0.2.12:46652] AH00128: File does not exist: /xxx/~ diff --git a/testcases/files/logs/apache-noscript b/testcases/files/logs/apache-noscript index 5d5d35ff..172bb510 100644 --- a/testcases/files/logs/apache-noscript +++ b/testcases/files/logs/apache-noscript @@ -1 +1,4 @@ +# failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" } [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat +# failJSON: { "time": "2005-07-22T06:48:30", "match": true , "host": "198.51.100.86" } +[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php diff --git a/testcases/files/logs/apache-overflows b/testcases/files/logs/apache-overflows index 18a44bfc..1af377f1 100644 --- a/testcases/files/logs/apache-overflows +++ b/testcases/files/logs/apache-overflows @@ -1,2 +1,4 @@ +# failJSON: { "time": "2005-03-16T15:39:29", "match": true , "host": "58.179.109.179" } [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 +# failJSON: { "time": "2005-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 diff --git a/testcases/files/logs/assp b/testcases/files/logs/assp index 99363001..2c658eb9 100644 --- a/testcases/files/logs/assp +++ b/testcases/files/logs/assp @@ -1,13 +1,25 @@ +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:10:37", "match": true , "host": "68.171.223.68" } Apr-07-13 07:10:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:12:37", "match": true , "host": "68.171.223.68" } Apr-07-13 07:12:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:14:36", "match": true , "host": "68.171.223.68" } Apr-07-13 07:14:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (8); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (9); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (10); +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" } Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; diff --git a/testcases/files/logs/asterisk b/testcases/files/logs/asterisk index 45b69304..ec2fec7c 100644 --- a/testcases/files/logs/asterisk +++ b/testcases/files/logs/asterisk @@ -1,15 +1,40 @@ # Sample log files for asterisk +# failJSON: { "time": "2012-02-13T17:21:54", "match": true , "host": "1.2.3.4" } [2012-02-13 17:21:54] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Wrong password +# failJSON: { "time": "2012-02-13T17:18:22", "match": true , "host": "1.2.3.4" } [2012-02-13 17:18:22] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - No matching peer found +# failJSON: { "time": "2012-02-13T17:21:21", "match": true , "host": "1.2.3.4" } [2012-02-13 17:21:21] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Username/auth name mismatch +# failJSON: { "time": "2012-02-13T17:32:01", "match": true , "host": "1.2.3.4" } [2012-02-13 17:32:01] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Device does not match ACL +# failJSON: { "time": "2012-02-13T17:34:10", "match": true , "host": "1.2.3.4" } [2012-02-13 17:34:10] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - Peer is not supposed to register +# failJSON: { "time": "2012-02-13T17:36:23", "match": true , "host": "1.2.3.4" } [2012-02-13 17:36:23] NOTICE[1638] chan_sip.c: Registration from '' failed for '1.2.3.4' - ACL error (permit/deny) +# failJSON: { "time": "2012-02-13T17:53:59", "match": true , "host": "1.2.3.4" } [2012-02-13 17:53:59] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed to authenticate as 'Fail2ban' +# failJSON: { "time": "2012-02-13T17:39:20", "match": true , "host": "1.2.3.4" } [2012-02-13 17:39:20] NOTICE[1638] chan_iax2.c: No registration for peer 'Fail2ban' (from 1.2.3.4) +# failJSON: { "time": "2012-02-13T17:44:26", "match": true , "host": "1.2.3.4" } [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) +# failJSON: { "time": "2012-02-13T17:37:07", "match": true , "host": "1.2.3.4" } [2012-02-13 17:37:07] NOTICE[1638] chan_sip.c: Failed to authenticate user "Fail2ban" ;tag=1r698745234 +# failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. +# failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found +# failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" } [2013-05-13 07:10:53] SECURITY[1204] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="1368439853-500975",Severity="Error",Service="SIP",EventVersion="1",AccountID="00972599580679",SessionID="0x7f8ecc0421f8",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/1.2.3.4/5070" +# failJSON: { "time": "2013-06-10T18:15:03", "match": true , "host": "1.2.3.4" } [2013-06-10 18:15:03] NOTICE[2723] chan_sip.c: Registration from '"100"' failed for '1.2.3.4' - Not a local domain +# http://forum.4psa.com/showthread.php?t=6601 +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } +[2009-12-22 16:35:24] NOTICE[6163] chan_sip.c: Sending fake auth rejection for device ;tag=e3793a95e1acbc69o +# http://www.freepbx.org/forum/general-help/fake-auth-rejection +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } +[2009-12-22 16:35:24] NOTICE[1570][C-00000086] chan_sip.c: Sending fake auth rejection for device 1022;tag=5d8b6f92 +# http://www.spinics.net/lists/asterisk/msg127381.html +# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" } +[2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user ;tag=6pwd6erg54 +# failJSON: { "time": "2013-07-06T09:09:25", "match": true , "host": "141.255.164.106" } +[2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39" diff --git a/testcases/files/logs/courierlogin b/testcases/files/logs/courierlogin new file mode 100644 index 00000000..e3d0d8c3 --- /dev/null +++ b/testcases/files/logs/courierlogin @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-23T21:59:01", "match": true , "host": "1.2.3.4" } +Apr 23 21:59:01 dns2 imapd: LOGIN FAILED, user=sales@example.com, ip=[::ffff:1.2.3.4] +# failJSON: { "time": "2005-04-23T21:59:38", "match": true , "host": "198.51.100.76" } +Apr 23 21:59:38 dns2 pop3d: LOGIN FAILED, user=info@example.com, ip=[::ffff:198.51.100.76] +# failJSON: { "time": "2004-11-13T08:11:53", "match": true , "host": "198.51.100.33" } +Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff:198.51.100.33] +# failJSON: { "time": "2005-04-17T19:17:11", "match": true , "host": "1.2.3.4" } +Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4] diff --git a/testcases/files/logs/couriersmtp b/testcases/files/logs/couriersmtp new file mode 100644 index 00000000..212df3b4 --- /dev/null +++ b/testcases/files/logs/couriersmtp @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-04-10T03:47:57", "match": true , "host": "1.2.3.4" } +Apr 10 03:47:57 web courieresmtpd: error,relay=::ffff:1.2.3.4,ident=tmf,from=,to=: 550 User unknown. +# failJSON: { "time": "2005-07-06T03:42:28", "match": true , "host": "1.2.3.4" } +Jul 6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=: 550 User unknown. +# failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } +Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. +# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. diff --git a/testcases/files/logs/cyrus-imap b/testcases/files/logs/cyrus-imap new file mode 100644 index 00000000..9bf271f6 --- /dev/null +++ b/testcases/files/logs/cyrus-imap @@ -0,0 +1,13 @@ +# failJSON: { "time": "2005-01-04T21:51:05", "match": true , "host": "127.0.0.1" } +Jan 4 21:51:05 hostname cyrus/imap[5355]: badlogin: localhost.localdomain [127.0.0.1] plaintext cyrus@localdomain SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "198.51.100.23" } +Feb 20 17:23:32 domain cyrus/pop3[18635]: badlogin: localhost [198.51.100.23] plaintext administrator SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "1.2.3.4" } +Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail0001 SASL(-13): authentication failure: checkpass failed +# failJSON: { "time": "2005-06-08T18:11:13", "match": true , "host": "198.51.100.45" } +Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated] +# failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" } +Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response] +# failJSON: { "time": "2004-12-30T16:03:27", "match": true , "host": "1.2.3.4" } +Dec 30 16:03:27 somehost imapd[2517]: badlogin: local-somehost[1.2.3.4] OTP [SASL(-13): authentication failure: External SSF not good enough] + diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index f904a8fe..553ed621 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1,15 +1,31 @@ +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" } @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" } +@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root +# failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 +# failJSON: { "time": "2005-06-13T16:30:54", "match": true , "host": "49.176.98.87" } Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected +# failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } +Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 +# failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" } +Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= +# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" } +Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= diff --git a/testcases/files/logs/dropbear b/testcases/files/logs/dropbear new file mode 100644 index 00000000..139292d7 --- /dev/null +++ b/testcases/files/logs/dropbear @@ -0,0 +1,9 @@ +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 +# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } +Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 +# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" } +Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495 + diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim index d86b6455..95c70d25 100644 --- a/testcases/files/logs/exim +++ b/testcases/files/logs/exim @@ -1,6 +1,31 @@ # From IRC 2013-01-04 +# failJSON: { "time": "2013-01-04T17:03:46", "match": true , "host": "24.106.174.74" } 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) # From IRC 2013-06-13 XATRIX (Georgiy Mernov) +# failJSON: { "time": "2013-06-12T03:57:58", "match": true , "host": "120.196.140.45" } 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) +# failJSON: { "time": "2013-06-12T13:18:11", "match": true , "host": "101.66.165.86" } 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data +# failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "193.169.56.211" } 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address +# http://forum.lissyara.su/viewtopic.php?f=20&t=29857 +# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html +# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# failJSON: { "time": "2013-06-10T18:33:32", "match": true , "host": "202.132.70.178" } +2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted +# failJSON: { "time": "2013-06-02T06:54:20", "match": true , "host": "211.148.195.192" } +2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n" +# failJSON: { "time": "2013-06-02T09:05:48", "match": true , "host": "82.96.160.77" } +2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n" +# failJSON: { "time": "2013-06-02T09:22:05", "match": true , "host": "163.14.21.161" } +2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET") +# failJSON: { "time": "2013-06-02T15:06:18", "match": true , "host": "46.20.35.114" } +2013-06-02 15:06:18 H=(VM-WIN2K3-1562) [46.20.35.114] sender verify fail for : Unknown user +# failJSON: { "time": "2013-06-07T02:02:09", "match": true , "host": "91.232.21.92" } +2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for : all relevant MX records point to non-existent hosts +# failJSON: { "time": "2013-06-15T16:34:55", "match": true , "host": "182.18.24.93" } +2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F= rejected RCPT : Sender verify failed +# failJSON: { "time": "2013-06-15T16:36:49", "match": true , "host": "111.67.203.116" } +2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F= rejected RCPT : Unknown user + diff --git a/testcases/files/logs/exim-spam b/testcases/files/logs/exim-spam new file mode 100644 index 00000000..46b915d1 --- /dev/null +++ b/testcases/files/logs/exim-spam @@ -0,0 +1,17 @@ +# http://forum.lissyara.su/viewtopic.php?f=20&t=29857 +# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem +# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html +# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory +# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html +# failJSON: { "time": "2004-01-18T07:15:35", "match": true , "host": "4.47.28.40" } +2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0 +# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875 +# failJSON: { "time": "2013-06-15T11:19:33", "match": true , "host": "2.181.148.95" } +2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org +# failJSON: { "time": "2013-06-09T10:21:28", "match": true , "host": "46.254.240.82" } +2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected +# failJSON: { "time": "2013-06-15T11:20:36", "match": true , "host": "83.235.177.148" } +2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL). +# failJSON: { "time": "2013-06-16T02:50:43", "match": true , "host": "111.67.203.114" } +2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F= rejected RCPT : rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114 + diff --git a/testcases/files/logs/gssftpd b/testcases/files/logs/gssftpd new file mode 100644 index 00000000..58fda7dc --- /dev/null +++ b/testcases/files/logs/gssftpd @@ -0,0 +1,2 @@ +# failJSON: { "time": "2005-01-22T18:09:46", "match": true , "host": "198.51.100.23" } +Jan 22 18:09:46 host ftpd[132]: repeated login failures from 198.51.100.23 (example.com) diff --git a/testcases/files/logs/lighttpd b/testcases/files/logs/lighttpd-auth similarity index 60% rename from testcases/files/logs/lighttpd rename to testcases/files/logs/lighttpd-auth index c3cfcb75..745a2462 100644 --- a/testcases/files/logs/lighttpd +++ b/testcases/files/logs/lighttpd-auth @@ -1,3 +1,5 @@ #authentification failure (mod_auth) +# failJSON: { "time": "2011-12-25T17:09:20", "match": true , "host": "4.4.4.4" } 2011-12-25 17:09:20: (http_auth.c.875) password doesn't match for /gitweb/ username: francois, IP: 4.4.4.4 +# failJSON: { "time": "2012-09-26T10:24:35", "match": true , "host": "4.4.4.4" } 2012-09-26 10:24:35: (http_auth.c.1136) digest: auth failed for xxx : wrong password, IP: 4.4.4.4 diff --git a/testcases/files/logs/mysqld.log b/testcases/files/logs/mysqld-auth similarity index 51% rename from testcases/files/logs/mysqld.log rename to testcases/files/logs/mysqld-auth index b3a73078..757ecb66 100644 --- a/testcases/files/logs/mysqld.log +++ b/testcases/files/logs/mysqld-auth @@ -1,6 +1,12 @@ +# failJSON: { "time": "2013-03-24T00:04:00", "match": true , "host": "192.168.1.35" } 130324 0:04:00 [Warning] Access denied for user 'root'@'192.168.1.35' (using password: NO) +# failJSON: { "time": "2013-03-24T08:24:09", "match": true , "host": "220.95.238.171" } 130324 8:24:09 [Warning] Access denied for user 'root'@'220.95.238.171' (using password: NO) +# failJSON: { "time": "2013-03-24T17:56:13", "match": true , "host": "61.160.223.112" } 130324 17:56:13 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: NO) +# failJSON: { "time": "2013-03-24T17:56:14", "match": true , "host": "61.160.223.112" } 130324 17:56:14 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: YES) +# failJSON: { "time": "2013-03-24T19:01:39", "match": true , "host": "61.147.108.35" } 130324 19:01:39 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: NO) +# failJSON: { "time": "2013-03-24T19:01:40", "match": true , "host": "61.147.108.35" } 130324 19:01:40 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: YES) diff --git a/testcases/files/logs/named-refused b/testcases/files/logs/named-refused index 130e7417..6f6092e2 100644 --- a/testcases/files/logs/named-refused +++ b/testcases/files/logs/named-refused @@ -1,6 +1,12 @@ +# failJSON: { "time": "2005-07-24T14:16:55", "match": true , "host": "194.145.196.18" } Jul 24 14:16:55 raid5 named[3935]: client 194.145.196.18#4795: query 'ricreig.com/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:16:56", "match": true , "host": "62.123.164.113" } Jul 24 14:16:56 raid5 named[3935]: client 62.123.164.113#32768: query 'ricreig.com/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:17:13", "match": true , "host": "148.160.29.6" } Jul 24 14:17:13 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'geo-mueller.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:20:25", "match": true , "host": "148.160.29.6" } Jul 24 14:20:25 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'shivaree.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "148.160.29.6" } Jul 24 14:23:36 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'mietberatung.de/NS/IN' denied +# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "62.109.4.89" } Jul 24 14:23:36 raid5 named[3935]: client 62.109.4.89#9334: view external: query (cache) './NS/IN' denied diff --git a/testcases/files/logs/pam-generic b/testcases/files/logs/pam-generic index d84ab153..dc7efeee 100644 --- a/testcases/files/logs/pam-generic +++ b/testcases/files/logs/pam-generic @@ -1,7 +1,14 @@ +# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1" } Feb 7 15:10:42 example pure-ftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=pure-ftpd ruser=sample-user rhost=192.168.1.1 +# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } May 12 09:47:54 vaio sshd[16004]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com user=root +# failJSON: { "time": "2005-05-12T09:48:03", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } May 12 09:48:03 vaio sshd[16021]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com +# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62" } May 15 18:02:12 localhost proftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=66.232.129.62 user=mark +# failJSON: { "time": "2004-11-25T17:12:13", "match": true , "host": "192.168.10.3" } Nov 25 17:12:13 webmail pop(pam_unix)[4920]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.168.10.3 user=mailuser +# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" } Jul 19 18:11:26 srv2 vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com +# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" } Jul 19 18:11:26 srv2 vsftpd: pam_unix: authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com diff --git a/testcases/files/logs/php-url-fopen b/testcases/files/logs/php-url-fopen new file mode 100644 index 00000000..f119a928 --- /dev/null +++ b/testcases/files/logs/php-url-fopen @@ -0,0 +1,2 @@ +# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" } +66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 28ff95a4..122ad8e5 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -1,3 +1,12 @@ # per https://github.com/fail2ban/fail2ban/issues/125 # and https://github.com/fail2ban/fail2ban/issues/126 +# failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" } Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= +# failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" } +Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 : Relay access denied; from= to= proto=SMTP helo=<198.51.100.17> +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.65" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 : Helo command rejected: match bad.domain; from= to= proto=SMTP helo= +# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" } +Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> +# failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" } +Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab diff --git a/testcases/files/logs/proftpd b/testcases/files/logs/proftpd index aaaf0295..9687d992 100644 --- a/testcases/files/logs/proftpd +++ b/testcases/files/logs/proftpd @@ -1,9 +1,16 @@ +# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" } Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers +# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" } Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 +# failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" } Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. +# failJSON: { "time": "2005-06-09T11:15:43", "match": true , "host": "101.71.143.238" } Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21 +# failJSON: { "time": "2005-06-13T22:07:23", "match": true , "host": "59.167.242.100" } Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. +# failJSON: { "time": "2005-06-14T00:09:59", "match": true , "host": "59.167.242.100" } Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 - - - +# failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } +May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 diff --git a/testcases/files/logs/pure-ftpd b/testcases/files/logs/pure-ftpd index 4b4e3455..ef49b2ff 100644 --- a/testcases/files/logs/pure-ftpd +++ b/testcases/files/logs/pure-ftpd @@ -1,2 +1,4 @@ +# failJSON: { "time": "2005-01-31T16:54:07", "match": true , "host": "24.79.92.194" } Jan 31 16:54:07 desktop pure-ftpd: (?@24.79.92.194) [WARNING] Authentication failed for user [Administrator] +# failJSON: { "time": "2004-11-05T18:54:02", "match": true , "host": "server202181210195.ixlink.net" } Nov 5 18:54:02 pure-ftpd: (?@server202181210195.ixlink.net) [WARNING] Authentication failed for user [Administrator] diff --git a/testcases/files/logs/qmail b/testcases/files/logs/qmail new file mode 100644 index 00000000..63cad4cc --- /dev/null +++ b/testcases/files/logs/qmail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2004-09-06T07:33:33", "match": true , "host": "198.51.100.77" } +Sep 6 07:33:33 sd6 qmail: 1157520813.485077 rblsmtpd: 198.51.100.77 pid 19597 sbl-xbl.spamhaus.org: 451 http://www.spamhaus.org/query/bl?ip=198.51.100.77 +# failJSON: { "time": "2004-09-06T07:18:29", "match": true , "host": "198.51.100.54" } +Sep 6 07:18:29 sd6 qmail: 1157519909.633171 qmail-smtpd: 421 badiprbl: ip 198.51.100.54 rbl: example.com diff --git a/testcases/files/logs/recidive b/testcases/files/logs/recidive new file mode 100644 index 00000000..cf6df933 --- /dev/null +++ b/testcases/files/logs/recidive @@ -0,0 +1,6 @@ +# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [sendmail] Ban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 16:07:31,183 fail2ban.actions: WARNING [sendmail] Unban 1.2.3.4 +# failJSON: { "match": false } +2006-02-13 15:52:30,388 fail2ban.actions: WARNING [recidive] Ban 1.2.3.4 diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index 04e0faf5..7c16efbd 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -1,2 +1,6 @@ +# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" } [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 +# failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" } May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) diff --git a/testcases/files/logs/sasl b/testcases/files/logs/sasl index 18c5ff14..ca306098 100644 --- a/testcases/files/logs/sasl +++ b/testcases/files/logs/sasl @@ -1,5 +1,7 @@ #1 Example from postfix from dbts #507990 +# failJSON: { "time": "2004-12-02T22:24:22", "match": true , "host": "114.44.142.233" } Dec 2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.net[114.44.142.233]: SASL CRAM-MD5 authentication failed: PDc3OTEwNTkyNTEyMzA2NDIuMTIyODI1MzA2MUBoZWw+ #2 Example from postfix from dbts #573314 +# failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" } Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure diff --git a/testcases/files/logs/sieve b/testcases/files/logs/sieve new file mode 100644 index 00000000..770d17ad --- /dev/null +++ b/testcases/files/logs/sieve @@ -0,0 +1,6 @@ +# failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" } +Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2005-07-18T17:21:58", "match": true , "host": "1.2.3.4" } +Jul 18 17:21:58 ophelia cyrus/timsieved[12305]: badlogin: example.com[1.2.3.4] PLAIN authentication failure +# failJSON: { "time": "2004-09-25T22:07:38", "match": true , "host": "1.2.3.4" } +Sep 25 22:07:38 web9 timsieved[21040]: badlogin: web4[1.2.3.4] OTP authentication failure diff --git a/testcases/files/logs/sogo-auth b/testcases/files/logs/sogo-auth index e4fb9409..02a69c6d 100644 --- a/testcases/files/logs/sogo-auth +++ b/testcases/files/logs/sogo-auth @@ -1,17 +1,31 @@ # yoh: Kept original apache log lines as well, just in case they might come useful # for (testing) multiline regular expressions which would further constraint # SOGo log lines +# failJSON: { "match": false } Mar 24 08:58:32 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=hack0r,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:32", "match": true , "host": "173.194.44.31" } Mar 24 08:58:32 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'hack0r' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:32 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/38 0.311 - - 2M +# failJSON: { "match": false } Mar 24 08:58:40 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=kiddy,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:40", "match": true , "host": "173.194.44.31" } Mar 24 08:58:40 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'kiddy' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:40 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/37 0.007 - - 32K +# failJSON: { "match": false } Mar 24 08:58:50 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=plsBanMe,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:50", "match": true , "host": "173.194.44.31" } Mar 24 08:58:50 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'plsBanMe' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:50 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/40 0.008 - - 0 +# failJSON: { "match": false } Mar 24 08:58:59 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=root,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:58:59", "match": true , "host": "173.194.44.31" } Mar 24 08:58:59 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'root' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "match": false } 173.194.44.31 - - [24/Mar/2013:08:58:59 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/36 0.007 - - 0 +# failJSON: { "match": false } Mar 24 08:59:04 sogod [26818]: <0x0xb8537990[LDAPSource]> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=admin,ou=users,dc=mail,dc=example,dc=org"; } +# failJSON: { "time": "2005-03-24T08:59:04", "match": true , "host": "173.194.44.31" } Mar 24 08:59:04 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'admin' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 diff --git a/testcases/files/logs/sshd b/testcases/files/logs/sshd index 5dab1606..75854774 100644 --- a/testcases/files/logs/sshd +++ b/testcases/files/logs/sshd @@ -1,48 +1,87 @@ #1 +# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.030.0.6" } Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6 +# failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" } May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com #2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345 #3 +# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 +# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 #4 +# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "211.114.51.213" } Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 211.114.51.213 #5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch # yoh: added ':' after [sshd] since the case without is not really common any more +# failJSON: { "time": "2005-03-03T00:17:22", "match": true , "host": "211.188.220.49" } Mar 3 00:17:22 [sshd]: User root from 211.188.220.49 not allowed because not listed in AllowUsers +# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "example.com" } Feb 25 14:34:11 belka sshd[31607]: User root from example.com not allowed because not listed in AllowUsers #6 ew filter introduced thanks to report Guido Bozzetto +# failJSON: { "time": "2004-11-11T23:33:27", "match": true , "host": "218.249.210.161" } Nov 11 23:33:27 Server sshd[5174]: refused connect from _U2FsdGVkX19P3BCJmFBHhjLza8BcMH06WCUVwttMHpE=_@::ffff:218.249.210.161 (::ffff:218.249.210.161) #7 added exclamation mark to BREAK-IN # Now should be a negative since we decided not to catch those +# failJSON: { "match": false } Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT +# failJSON: { "match": false } Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! #8 DenyUsers https://github.com/fail2ban/fail2ban/issues/47 +# failJSON: { "time": "2005-04-16T22:01:15", "match": true , "host": "46.45.128.3" } Apr 16 22:01:15 al-ribat sshd[5154]: User root from 46.45.128.3 not allowed because listed in DenyUsers #9 OpenSolaris patch - pull https://github.com/fail2ban/fail2ban/pull/182 +# failJSON: { "time": "2005-03-29T05:59:23", "match": true , "host": "205.186.180.55" } Mar 29 05:59:23 dusky sshd[20878]: [ID 800047 auth.info] Failed keyboard-interactive for from 205.186.180.55 port 42742 ssh2 +# failJSON: { "time": "2005-03-29T05:20:09", "match": true , "host": "205.186.180.30" } Mar 29 05:20:09 dusky sshd[19558]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.30 port 54520 ssh2 #10 OSX syslog error +# failJSON: { "time": "2005-04-29T17:16:20", "match": true , "host": "example.com" } Apr 29 17:16:20 Jamess-iMac.local sshd[62312]: error: PAM: authentication error for james from example.com via 192.168.1.201 +# failJSON: { "time": "2005-04-29T20:11:08", "match": true , "host": "205.186.180.35" } Apr 29 20:11:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for from 205.186.180.35 port 42742 ssh2 +# failJSON: { "time": "2005-04-29T20:12:08", "match": true , "host": "205.186.180.22" } Apr 29 20:12:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.22 port 54520 ssh2 +# failJSON: { "time": "2005-04-29T20:13:08", "match": true , "host": "205.186.180.42" } Apr 29 20:13:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for james from 205.186.180.42 port 54520 ssh2 +# failJSON: { "time": "2005-04-29T20:14:08", "match": true , "host": "205.186.180.44" } Apr 29 20:14:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for from 205.186.180.44 port 42742 ssh2 +# failJSON: { "time": "2005-04-30T01:42:12", "match": true , "host": "205.186.180.77" } Apr 30 01:42:12 Jamess-iMac.local sshd[2554]: Failed keyboard-interactive/pam for invalid user jamedds from 205.186.180.77 port 33723 ssh2 +# failJSON: { "time": "2005-04-29T12:53:38", "match": true , "host": "205.186.180.88" } Apr 29 12:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication failure for james from 205.186.180.88 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T13:53:38", "match": true , "host": "205.186.180.99" } Apr 29 13:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication failure for james from 205.186.180.99 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T15:53:38", "match": true , "host": "205.186.180.100" } Apr 29 15:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication error for james from 205.186.180.100 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T16:53:38", "match": true , "host": "205.186.180.101" } Apr 29 16:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.101 via 192.168.1.201 +# failJSON: { "time": "2005-04-29T17:53:38", "match": true , "host": "205.186.180.102" } Apr 29 17:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.102 +# failJSON: { "time": "2005-04-29T18:53:38", "match": true , "host": "205.186.180.103" } Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.103 + +#11 https://github.com/fail2ban/fail2ban/issues/267 There might be no colon after [daemon] +# failJSON: { "time": "2005-06-25T23:53:34", "match": true , "host": "1.2.3.4" } +Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers +# failJSON: { "time": "2004-12-12T20:04:39", "match": true , "host": "10.215.4.227" } +Dec 12 20:04:39 aragorn sshd[1328]: error: PAM: User not known to the underlying authentication module for illegal user kernelitshell from 10.215.4.227 +# failJSON: { "time": "2005-03-26T04:56:27", "match": true , "host": "example.com" } +Mar 26 04:56:27 angel sshd[9739]: User allena from example.com not allowed because not in any group +# failJSON: { "time": "2005-02-07T16:01:07", "match": true , "host": "192.51.100.54" } +Feb 7 16:01:07 linux-m899 sshd[5106]: User root from 192.51.100.54 not allowed because a group is listed in DenyGroups +# failJSON: { "time": "2005-01-05T11:15:05", "match": true , "host": "10.0.0.40" } +Jan 5 11:15:05 NAS sshd[1966]: User root from 10.0.0.40 not allowed because none of user's groups are listed in AllowGroups diff --git a/testcases/files/logs/sshd-ddos b/testcases/files/logs/sshd-ddos index d71c6bb2..452abbde 100644 --- a/testcases/files/logs/sshd-ddos +++ b/testcases/files/logs/sshd-ddos @@ -1,2 +1,3 @@ # http://forums.powervps.com/showthread.php?t=1667 +# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" } Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 diff --git a/testcases/files/logs/suhosin b/testcases/files/logs/suhosin new file mode 100644 index 00000000..90ed7bf1 --- /dev/null +++ b/testcases/files/logs/suhosin @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-03-11T22:52:12", "match": true , "host": "198.51.100.167" } +Mar 11 22:52:12 lighttpd[53690]: (mod_fastcgi.c.2676) FastCGI-stderr: ALERT - configured request variable name length limit exceeded - dropped variable 'upqchi07vFfAFuBjnIKGIwiLrHo3Vt68T3yqvhQu2TqetQ78roy7Q6bpTfDUtYFR593/MA' (attacker '198.51.100.167', file '/usr/local/captiveportal/index.php') +# failJSON: { "time": "2005-02-26T22:52:29", "match": true , "host": "198.51.100.77" } +Feb 26 22:52:29 host suhosin[9636]: ALERT - script tried to increase memory_limit to 268435456 bytes which is above the allowed value (attacker '198.51.100.77', file '/var/www/wordpress/wp-admin/includes/image.php', line 161) diff --git a/testcases/files/logs/vsftpd b/testcases/files/logs/vsftpd index a8b6a4cf..f3fb997f 100644 --- a/testcases/files/logs/vsftpd +++ b/testcases/files/logs/vsftpd @@ -1,7 +1,10 @@ #1 PAM based +# failJSON: { "time": "2004-10-11T01:06:47", "match": true , "host": "209.67.1.67" } Oct 11 01:06:47 ServerJV vsftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=209.67.1.67 +# failJSON: { "time": "2005-02-06T12:02:29", "match": true , "host": "64.168.103.1" } Feb 6 12:02:29 server vsftpd(pam_unix)[15522]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=64.168.103.1 user=user1 #2 Internal +# failJSON: { "time": "2005-01-19T12:20:33", "match": true , "host": "64.106.46.98" } Fri Jan 19 12:20:33 2007 [pid 27202] [anonymous] FAIL LOGIN: Client "64.106.46.98" diff --git a/testcases/files/logs/webmin-auth b/testcases/files/logs/webmin-auth index 86d3f321..424c114d 100644 --- a/testcases/files/logs/webmin-auth +++ b/testcases/files/logs/webmin-auth @@ -1,7 +1,9 @@ #Webmin authentication failures from /var/log/auth.log #1 User exists, bad password +# failJSON: { "time": "2004-12-13T08:15:18", "match": true , "host": "89.2.49.230" } Dec 13 08:15:18 sb1 webmin[25875]: Invalid login as root from 89.2.49.230 #2 User does not exists +# failJSON: { "time": "2004-12-12T23:14:19", "match": true , "host": "188.40.105.142" } Dec 12 23:14:19 sb1 webmin[22134]: Non-existent login as robert from 188.40.105.142 diff --git a/testcases/files/logs/wu-ftpd b/testcases/files/logs/wu-ftpd deleted file mode 100644 index b6b41613..00000000 --- a/testcases/files/logs/wu-ftpd +++ /dev/null @@ -1,2 +0,0 @@ -# This login line is from syslog -Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] diff --git a/testcases/files/logs/wuftpd b/testcases/files/logs/wuftpd new file mode 100644 index 00000000..bbb816cc --- /dev/null +++ b/testcases/files/logs/wuftpd @@ -0,0 +1,5 @@ +# This login line is from syslog +# failJSON: { "time": "2004-10-06T09:59:26", "match": true , "host": "202.108.145.173" } +Oct 6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173] +# failJSON: { "time": "2004-10-11T16:45:07", "match": true , "host": "198.51.100.71" } +Oct 11 16:45:07 ubuntu wu-ftpd[2360]: failed login from example.com [198.51.100.71] diff --git a/testcases/files/logs/xinetd-fail b/testcases/files/logs/xinetd-fail new file mode 100644 index 00000000..8545a067 --- /dev/null +++ b/testcases/files/logs/xinetd-fail @@ -0,0 +1,4 @@ +# failJSON: { "time": "2005-05-15T17:38:49", "match": true , "host": "198.51.100.169" } +May 15 17:38:49 boo xinetd[16256]: FAIL: telnet address from=198.51.100.169 +# failJSON: { "time": "2005-08-03T14:38:49", "match": true , "host": "198.51.100.223" } +Aug 3 14:38:49 backup xinetd[31234]: FAIL: amanda libwrap from=198.51.100.223 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 1657f694..d918bbf7 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -39,6 +39,8 @@ from server.failmanager import FailManagerEmpty # Useful helpers # +from utils import mtimesleep + # yoh: per Steven Hiscocks's insight while troubleshooting # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 # adding a sufficiently large buffer might help to guarantee that @@ -68,18 +70,6 @@ def _killfile(f, name): _killfile(None, name + '.bak') -def _sleep_4_poll(): - """PollFilter relies on file timestamps - so we might need to - sleep to guarantee that they differ - """ - if sys.version_info[:2] <= (2,4): - # on old Python st_mtime is int, so we should give - # at least 1 sec so polling filter could detect - # the change - time.sleep(1.) - else: - time.sleep(0.1) - def _assert_equal_entries(utest, found, output, count=None): """Little helper to unify comparisons with the target entries @@ -119,10 +109,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line Returns open fout """ - if sys.version_info[:2] <= (2,4): # pragma: no cover - # on old Python st_mtime is int, so we should give at least 1 sec so - # polling filter could detect the change - time.sleep(1) + # on older Pythons and on some FSs st_mtime is int, so we should + # give it some time so polling filter could detect the change + mtimesleep() if isinstance(fin, str): # pragma: no branch - only used with str in test cases fin = open(fin, 'r') # Skip @@ -237,14 +226,14 @@ class LogFileMonitor(unittest.TestCase): # but not any longer self.assertTrue(self.notModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() # to guarantee freshier mtime + mtimesleep() # to guarantee freshier mtime for i in range(4): # few changes # unless we write into it self.file.write("line%d\n" % i) self.file.flush() self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() # to guarantee freshier mtime + mtimesleep() # to guarantee freshier mtime os.rename(self.name, self.name + '.old') # we are not signaling as modified whenever # it gets away @@ -252,7 +241,7 @@ class LogFileMonitor(unittest.TestCase): f = open(self.name, 'a') self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) - _sleep_4_poll() + mtimesleep() f.write("line%d\n" % i) f.flush() self.assertTrue(self.isModified()) @@ -398,7 +387,7 @@ def get_monitor_failures_testcase(Filter_): # actions might be happening too fast in the tests, # sleep a bit to guarantee reliable time stamps if isinstance(self.filter, FilterPoll): - _sleep_4_poll() + mtimesleep() def isEmpty(self, delay=0.4): # shorter wait time for not modified status @@ -458,7 +447,9 @@ def get_monitor_failures_testcase(Filter_): self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time - self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2)) + self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py new file mode 100644 index 00000000..54afc864 --- /dev/null +++ b/testcases/samplestestcase.py @@ -0,0 +1,134 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__copyright__ = "Copyright (c) 2013 Steven Hiscocks" +__license__ = "GPL" + +import unittest, sys, os, fileinput, re, datetime, inspect + +if sys.version_info >= (2, 6): + import json +else: + import simplejson as json + next = lambda x: x.next() + +from server.filter import Filter +from client.filterreader import FilterReader + +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") +CONFIG_DIR = "config" + +class FilterSamplesRegex(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + self.filter = Filter(None) + self.filter.setActive(True) + + def tearDown(self): + """Call after every test case.""" + + def testFiltersPresent(self): + """Check to ensure some tests exist""" + self.assertTrue( + len([test for test in inspect.getmembers(self) + if test[0].startswith('testSampleRegexs')]) + >= 10, + "Expected more FilterSampleRegexs tests") + +def testSampleRegexsFactory(name): + def testFilter(self): + + # Check filter exists + filterConf = FilterReader(name, "jail", basedir=CONFIG_DIR) + filterConf.read() + filterConf.getOptions({}) + + for opt in filterConf.convert(): + if opt[2] == "addfailregex": + self.filter.addFailRegex(opt[3]) + elif opt[2] == "addignoreregex": + self.filter.addIgnoreRegex(opt[3]) + + if not self.filter.getFailRegex(): + # No fail regexs set: likely just common file for includes. + return + + self.assertTrue( + os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), + "No sample log file available for '%s' filter" % name) + + logFile = fileinput.FileInput( + os.path.join(TEST_FILES_DIR, "logs", name)) + + regexsUsed = set() + for line in logFile: + jsonREMatch = re.match("^# ?failJSON:(.+)$", line) + if jsonREMatch: + try: + faildata = json.loads(jsonREMatch.group(1)) + except ValueError, e: + raise ValueError("%s: %s:%i" % + (e, logFile.filename(), logFile.filelineno())) + line = next(logFile) + elif line.startswith("#") or not line.strip(): + continue + else: + faildata = {} + + ret = self.filter.processLine( + line, returnRawHost=True, checkAllRegex=True) + if not ret: + # Check line is flagged as none match + self.assertFalse(faildata.get('match', True), + "Line not matched when should have: %s:%i %r" % + (logFile.filename(), logFile.filelineno(), line)) + elif ret: + # Check line is flagged to match + self.assertTrue(faildata.get('match', False), + "Line matched when shouldn't have: %s:%i %r" % + (logFile.filename(), logFile.filelineno(), line)) + self.assertEqual(len(ret), 1, "Multiple regexs matched") + # Verify timestamp and host as expected + failregex, host, time = ret[0] + self.assertEqual(host, faildata.get("host", None)) + self.assertEqual( + datetime.datetime.fromtimestamp(time), + datetime.datetime.strptime( + faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")) + + regexsUsed.add(failregex) + + # TODO: Remove exception handling once all regexs have samples + for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): + self.assertTrue( + failRegexIndex in regexsUsed, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) + + return testFilter + +for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")): + filterName = filter_.rpartition(".")[0] + setattr( + FilterSamplesRegex, + "testSampleRegexs%s" % filterName.upper(), + testSampleRegexsFactory(filterName)) diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 0a5593e3..07cc373c 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -334,9 +334,9 @@ class Transmitter(TransmitterBase): "failed attempt from again", ], [ - "user john at (?:::f{4,6}:)?(?P[\w\-.^_]+)", - "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", - "failed attempt from (?:::f{4,6}:)?(?P[\w\-.^_]+) again", + "user john at (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", + "failed attempt from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w) again", ], self.jailName ) @@ -359,7 +359,7 @@ class Transmitter(TransmitterBase): ], [ "user john", - "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]*\\w)", "Dont match me!", ], self.jailName diff --git a/testcases/utils.py b/testcases/utils.py index 87aab915..643c9ad1 100644 --- a/testcases/utils.py +++ b/testcases/utils.py @@ -22,7 +22,7 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback +import logging, os, re, tempfile, sys, time, traceback from os.path import basename, dirname # @@ -100,3 +100,8 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + +def mtimesleep(): + # no sleep now should be necessary since polling tracks now not only + # mtime but also ino and size + pass