diff --git a/ChangeLog b/ChangeLog index 4deaf2de..2e255ea4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,10 +22,27 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) ### New Features +* new filter and jail for GitLab recognizing failed application logins (gh-2689) ### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..361cb177 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | cut -d'"' -f6) +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) + if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index a1065224..ab7cdac7 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,9 +27,9 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s actionunban = ipset del -exist @@ -44,9 +44,17 @@ chain = INPUT_direct # Option: default-timeout # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -default-timeout = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype # Notes.: defines additions to the blocking rule @@ -71,7 +79,7 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 # DEV NOTES: diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index c851233c..bf299259 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,9 +49,9 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -65,9 +65,17 @@ actionunban = ipset del -exist # Option: default-timeout # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -default-timeout = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -76,4 +84,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 12c3ddd6..e94751fa 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,9 +49,9 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -65,9 +65,17 @@ actionunban = ipset del -exist # Option: default-timeout # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -default-timeout = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -76,4 +84,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 45be0c0a..8cbbd347 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -66,9 +66,9 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -80,6 +80,14 @@ actionunban = ipset del f2b- -exist # Option: default-timeout # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -default-timeout = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=,.+$ +prefregex = ^%(__prefix_line)serror,relay=,(?:port=\d+,)?.+$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=.+ ip=$ diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index a7bd2837..d3f10b18 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -14,16 +14,15 @@ before = common.conf _daemon = proftpd -__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +__suffix_failed_login = ([uU]ser not authorized for login|[nN]o such user found|[iI]ncorrect password|[pP]assword expired|[aA]ccount disabled|[iI]nvalid shell: '\S+'|[uU]ser in \S+|[lL]imit (access|configuration) denies login|[nN]ot a UserAlias|[mM]aximum login length exceeded) -prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ (?:USER|SECURITY|Maximum).+$ +prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ (?:USER|SECURITY|Maximum) .+$ -failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^SECURITY VIOLATION: .* login attempted\. *$ - ^Maximum login attempts \(\d+\) exceeded *$ +failregex = ^USER \S+|.*?(?: \(Login failed\))?: %(__suffix_failed_login)s + ^SECURITY VIOLATION: \S+|.*? login attempted + ^Maximum login attempts \(\d+\) exceeded ignoreregex = diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 12631cb3..31e61b96 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -73,13 +73,12 @@ mdre-normal-other = ^(Connection closed|Disconnected) ^kex_exchange_identification: client sent invalid protocol identifier ^Bad protocol version identification '.*' from - ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer # same as mdre-normal-other, but as failure (without ) and [preauth] only: -mdre-ddos-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ +mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf index 8321a138..8022fee1 100644 --- a/config/filter.d/traefik-auth.conf +++ b/config/filter.d/traefik-auth.conf @@ -51,6 +51,26 @@ [Definition] -failregex = ^ \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^ \- > \[\] \"(?:) [^\"]+\" 401\b ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )\S+ +usrre-ddos = - +usrre-aggressive = \S+ \ No newline at end of file diff --git a/config/jail.conf b/config/jail.conf index c7177f13..fe4f0208 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,7 +60,7 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # @@ -212,19 +212,19 @@ banaction_allports = iptables-allports action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mwl = %(action_)s %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines @@ -371,7 +371,7 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log @@ -744,8 +744,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -756,9 +756,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -766,9 +765,8 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 @@ -856,8 +854,12 @@ logpath = /opt/cstrike/logs/L[0-9]*.log # Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log [bitwarden] port = http,https @@ -909,8 +911,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 80617a50..09b5fff0 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -53,13 +53,17 @@ class ActionReader(DefinitionInitConfigReader): } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 6b33d30a..0a4c84ed 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban ... ", "unbans (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned ... ]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -120,6 +122,8 @@ protocol = [ ["set action ", "sets the of for the action for "], ["set action [ ]", "calls the with for the action for "], ['', "JAIL INFORMATION", ""], +["get banned", "return banned IPs of "], +["get banned ... ]", "return 1 if IP is banned in otherwise 0, or a list of 1/0 for multiple IPs"], ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 40682bf3..9919a9e4 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -211,6 +211,14 @@ class Actions(JailThread, Mapping): def getBanTime(self): return self.__banManager.getBanTime() + def getBanned(self, ids): + lst = self.__banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + def getBanList(self, withTime=False): """Returns the list of banned IP addresses. diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 5770bfd7..a8dcecea 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -105,7 +105,7 @@ class BanManager: def getBanList(self, ordered=False, withTime=False): with self.__lock: if not ordered: - return self.__banList.keys() + return list(self.__banList.keys()) lst = [] for ticket in self.__banList.itervalues(): eob = ticket.getEndOfBanTime(self.__banTime) @@ -125,8 +125,9 @@ class BanManager: # @return ban list iterator def __iter__(self): + # ensure iterator is safe (traverse over the list in snapshot created within lock): with self.__lock: - return self.__banList.itervalues() + return iter(list(self.__banList.values())) ## # Returns normalized value diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8cf2be5b..25454ffd 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -81,6 +81,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -490,28 +491,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -551,8 +560,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 6648dac6..335fc473 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -379,6 +379,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index cfea85e0..4606d928 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -540,6 +540,32 @@ class Server: cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index f83e9d5f..1a3ee052 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -118,6 +118,9 @@ class Transmitter: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) elif name == "echo": return command[1:] elif name == "server-status": @@ -430,7 +433,10 @@ class Transmitter: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index d61a6520..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -57,11 +57,10 @@ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from ^%(__prefix_line_sl)sBad protocol version identification '.*' from - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index dd1796ae..e9da0e07 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -453,6 +453,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -1041,6 +1049,30 @@ class Fail2banServerTest(Fail2banClientServerBase): all=True) # if observer available wait for it becomes idle (write all tickets to db): _observer_wait_idle() + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # rotate logs: _write_file(test1log, "w+") diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..cea73073 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -12,3 +12,5 @@ Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..70ddc0e8 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 diff --git a/fail2ban/tests/files/logs/proftpd b/fail2ban/tests/files/logs/proftpd index b255e91e..8d0d571c 100644 --- a/fail2ban/tests/files/logs/proftpd +++ b/fail2ban/tests/files/logs/proftpd @@ -1,6 +1,6 @@ -# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers -# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 # failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" } Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. @@ -12,7 +12,9 @@ Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting. Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 # failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded -# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2004-10-02T15:45:44", "match": true , "host": "192.0.2.13", "user": "Root", "desc": "dot at end is optional (mod_sftp, gh-2246)" } +Oct 2 15:45:44 ftp01 proftpd[5517]: 192.0.2.13 (192.0.2.13[192.0.2.13]) - SECURITY VIOLATION: Root login attempted +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4", "user": "jtittle@domain.org" } Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 # failJSON: { "time": "2013-11-16T21:59:30", "match": true , "host": "1.2.3.4", "desc": "proftpd-basic 1.3.5~rc3-2.1 on Debian uses date format with milliseconds if logging under /var/log/proftpd/proftpd.log" } -2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21 +2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21 \ No newline at end of file diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 0385f38c..1bf9d913 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -296,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } @@ -327,6 +330,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth index 3e7a8987..edfe7306 100644 --- a/fail2ban/tests/files/logs/traefik-auth +++ b/fail2ban/tests/files/logs/traefik-auth @@ -1,6 +1,23 @@ +# filterOptions: [{"mode": "normal"}] + # failJSON: { "match": false } 10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "ddos"}] + +# failJSON: { "match": false } +10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}] + # failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" } 10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms +# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" } +10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms # failJSON: { "match": false } 10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" } +10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index c3b58408..585d8e96 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1899,7 +1899,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip4 = IPAddr('192.0.2.1') ip6 = IPAddr('2001:DB8::') self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) self.assertTrue(asip('192.0.2.1').isIPv4) self.assertTrue(id(asip(ip4)) == id(ip4)) @@ -1908,6 +1910,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1916,6 +1919,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1938,7 +1942,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertSortedEqual(res, []) - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1947,7 +1951,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1971,7 +1975,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: @@ -1993,8 +1997,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -2019,6 +2025,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -2114,7 +2122,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 9b986f53..43d76802 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -390,7 +390,15 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 8e944454..e379ccd1 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -224,7 +223,7 @@ class BanTimeIncrDB(LogCaptureTestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -385,10 +384,12 @@ class BanTimeIncrDB(LogCaptureTestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -477,7 +478,7 @@ class BanTimeIncrDB(LogCaptureTestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) failManager = FailManager() failManager.setMaxRetry(3) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 71347c7e..9bc90853 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1574,10 +1574,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain=""]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0`", + "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0 `", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( @@ -1597,23 +1597,23 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", default-timeout=0, chain=""]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0`", + "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( @@ -1633,13 +1633,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", @@ -1917,10 +1917,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0`", + "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( @@ -1940,27 +1940,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=, protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=, protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1976,13 +1976,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index fcfddba7..74b87fbf 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -331,13 +331,21 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), ('192.168.0.', set()), ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): @@ -552,7 +560,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -562,6 +570,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -580,8 +595,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 5e540545..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftabl [Service] Type=simple +Environment="PYTHONNOUSERSITE=1" ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index f02b8568..81473daa 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -111,6 +111,14 @@ jails and database) unbans (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned ... ]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -356,6 +364,14 @@ for .IP JAIL INFORMATION .TP +\fBget banned\fR +return banned IPs of +.TP +\fBget banned ... ]\fR +return 1 if IP is banned in +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget logpath\fR gets the list of the monitored files for