From 303861d7c752492a245bcf954fc4edf336b9e35d Mon Sep 17 00:00:00 2001 From: Mihail Politaev Date: Thu, 30 Jan 2020 21:17:32 +0200 Subject: [PATCH 001/139] Using native firewalld ipset implementation By creating additional action file firewallcmd-ipset-native.conf --- config/action.d/firewallcmd-ipset-native.conf | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 config/action.d/firewallcmd-ipset-native.conf diff --git a/config/action.d/firewallcmd-ipset-native.conf b/config/action.d/firewallcmd-ipset-native.conf new file mode 100644 index 00000000..757d46ad --- /dev/null +++ b/config/action.d/firewallcmd-ipset-native.conf @@ -0,0 +1,77 @@ +# Fail2Ban action file for firewall-cmd using native ipset implementation +# +# This requires: +# ipset (package: ipset) +# firewall-cmd (package: firewalld) +# +# This is for ipset protocol 6 (and hopefully later) (ipset v6.14). +# Use ipset -V to see the protocol and version. +# +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# +# If you are running on an older kernel you make need to patch in external +# modules. + +[INCLUDES] + +before = firewallcmd-common.conf + +[Definition] + +actionstart = firewall-cmd --permanent --new-ipset= --type=hash:ip --option=timeout= + firewall-cmd --reload + firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j + +actionstop = firewall-cmd --direct --remove-rule filter 0 -m set --match-set src -j + firewall-cmd --permanent --delete-ipset= + firewall-cmd --reload + +actionban = firewall-cmd --ipset= --add-entry= + +actionunban = firewall-cmd --ipset= --remove-entry= + +[Init] + +# Option: chain +# Notes specifies the iptables chain to which the fail2ban rules should be +# added +# Values: [ STRING ] +# +chain = INPUT_direct + +# Option: bantime +# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Values: [ NUM ] Default: 600 + +bantime = 86400 + +# Option: actiontype +# Notes.: defines additions to the blocking rule +# Values: leave empty to block all attempts from the host +# Default: Value of the multiport +actiontype = + +# Option: allports +# Notes.: default addition to block all ports +# Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] +# for all protocols: banaction = firewallcmd-ipset[actiontype=""] +allports = -p + +# Option: multiport +# Notes.: addition to block access only to specific ports +# Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] +multiport = -p -m multiport --dports + +ipmset = f2b- +familyopt = + +[Init?family=inet6] + +ipmset = f2b-6 +familyopt = family inet6 + + +# DEV NOTES: +# +# Author: Edgar Hoch and Daniel Black and Mihail Politaev +# firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness From ede200970859a6f1491aabdeb68c7d24a6052e36 Mon Sep 17 00:00:00 2001 From: Jordi Sanfeliu Date: Fri, 3 Apr 2020 12:52:19 +0200 Subject: [PATCH 002/139] added new jail (and filter) Monitorix --- config/filter.d/monitorix.conf | 27 +++++++++++++++++++++++++++ config/jail.conf | 5 +++++ 2 files changed, 32 insertions(+) create mode 100644 config/filter.d/monitorix.conf diff --git a/config/filter.d/monitorix.conf b/config/filter.d/monitorix.conf new file mode 100644 index 00000000..3979ed43 --- /dev/null +++ b/config/filter.d/monitorix.conf @@ -0,0 +1,27 @@ +# Fail2Ban filter for Monitorix (HTTP built-in server) +# + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = monitorix-httpd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P\S+) +# Values: TEXT +# +failregex = NOTEXIST - \[\] .* + AUTHERR - \[\] .* + NOTALLOWED - \[\] .* + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index c7177f13..02ce55f1 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -951,3 +951,8 @@ logpath = %(apache_error_log)s # see `filter.d/traefik-auth.conf` for details and service example. port = http,https logpath = /var/log/traefik/access.log + + +[monitorix] +port = 8080 +logpath = /var/log/monitorix-httpd From 412120ac3c3f1c9a3fa8c7676059ab2be0f87c6a Mon Sep 17 00:00:00 2001 From: aresdr Date: Sat, 30 May 2020 15:25:31 -0700 Subject: [PATCH 003/139] Update drupal-auth.conf Small fix for Drupal 8. D8 uses "Login attempt failed from" while D7 uses "Login attempt failed for". The referer part is a must currently, but some requests did not have one and are not failing. --- config/filter.d/drupal-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index b60abe3e..2d4cbe9f 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.+\|\d\|.*\|Login attempt failed for .+\.$ +failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.*\|\d\|.*\|Login attempt failed (?:for|from) .+\.$ ignoreregex = From 2216fd8da4e95564bc4cd0047ffae08d24d3d17d Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Tue, 4 Aug 2020 19:04:05 -0400 Subject: [PATCH 004/139] Add Apprise Support (50+ Notifications) --- MANIFEST | 1 + config/action.d/apprise.conf | 47 ++++++++++++++++++++++++++++++++++++ config/jail.conf | 9 +++++++ 3 files changed, 57 insertions(+) create mode 100644 config/action.d/apprise.conf diff --git a/MANIFEST b/MANIFEST index ed441bac..d02b8bd0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,6 +3,7 @@ bin/fail2ban-regex bin/fail2ban-server bin/fail2ban-testcases ChangeLog +config/action.d/apprise.conf config/action.d/abuseipdb.conf config/action.d/apf.conf config/action.d/badips.conf diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf new file mode 100644 index 00000000..ac54d8fd --- /dev/null +++ b/config/action.d/apprise.conf @@ -0,0 +1,47 @@ +# Fail2Ban configuration file +# +# Author: Chris Caron +# +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = printf %%b "The jail as been started successfully." |apprise -t "[Fail2Ban] : started on `uname -n`" -c /etc/fail2ban/apprise.conf + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = printf %%b "The jail has been stopped." |apprise -t "[Fail2Ban] : stopped on `uname -n`" -c /etc/fail2ban/apprise.conf + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | apprise -n "warning" -t "[Fail2Ban] : banned $ + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] + +# Define location of the default apprise configuration file to use +# +config = /etc/fail2ban/apprise.conf diff --git a/config/jail.conf b/config/jail.conf index 6e8a6a2f..dbca2e8f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -227,6 +227,15 @@ action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(proto action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[name=%(__name__)s, config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +action_apprise = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + apprise[name=%(__name__)s] + # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] From 70c601e9e53a44a6a2e950ee8e64705d14f09bc7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 2 Sep 2020 20:47:05 +0200 Subject: [PATCH 005/139] involve config parameter (replaces hard-coded path); fixed typo in actionban (looks like copy&paste from trimmed tty) --- config/action.d/apprise.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf index ac54d8fd..37c42ea2 100644 --- a/config/action.d/apprise.conf +++ b/config/action.d/apprise.conf @@ -10,13 +10,13 @@ # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = printf %%b "The jail as been started successfully." |apprise -t "[Fail2Ban] : started on `uname -n`" -c /etc/fail2ban/apprise.conf +actionstart = printf %%b "The jail as been started successfully." | -t "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = printf %%b "The jail has been stopped." |apprise -t "[Fail2Ban] : stopped on `uname -n`" -c /etc/fail2ban/apprise.conf +actionstop = printf %%b "The jail has been stopped." | -t "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -30,7 +30,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | apprise -n "warning" -t "[Fail2Ban] : banned $ +actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | -n "warning" -t "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -45,3 +45,5 @@ actionunban = # Define location of the default apprise configuration file to use # config = /etc/fail2ban/apprise.conf +# +apprise = apprise -c "" From 27c40a77a31c4c72b9076f43962debcf8d7d6359 Mon Sep 17 00:00:00 2001 From: stepodev Date: Wed, 25 Nov 2020 20:57:06 +0100 Subject: [PATCH 006/139] add nginx-tls-downgrade --- ChangeLog | 1 + config/filter.d/nginx-tls-fallback.conf | 16 ++++++++++++++++ config/jail.conf | 3 +++ fail2ban/tests/files/logs/nginx-tls-fallback | 9 +++++++++ 4 files changed, 29 insertions(+) create mode 100644 config/filter.d/nginx-tls-fallback.conf create mode 100644 fail2ban/tests/files/logs/nginx-tls-fallback diff --git a/ChangeLog b/ChangeLog index 5f1fc313..bd4a369d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) +* `filter.d/nginx-tls-fallback` -- filter added for tls downgrade probes ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools diff --git a/config/filter.d/nginx-tls-fallback.conf b/config/filter.d/nginx-tls-fallback.conf new file mode 100644 index 00000000..aaa84fb9 --- /dev/null +++ b/config/filter.d/nginx-tls-fallback.conf @@ -0,0 +1,16 @@ +# fail2ban filter configuration for nginx +# Ban people checking for TLS_FALLBACK_SCSV repeatedly +# https://stackoverflow.com/questions/28010492/nginx-critical-error-with-ssl-handshaking/28010608#28010608 + +[Definition] + + +failregex = ^ \[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed.*?ssl3_get_record.*?too.*?, client: , server: \S+$ + +ignoreregex = + +datepattern = {^LN-BEG} + +# Author: Stephan Orlowsky +# maybe not restrictive enough, will also match: +#"[crit] 76952#76952: *5062354 SSL_do_handshake() failed ssl3_get_record too, client: 0.0.0.0, server: thisshouldntmatch" diff --git a/config/jail.conf b/config/jail.conf index 28f259a0..a7818a06 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -397,6 +397,9 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_access_log)s +[nginx-tls-fallback] +port = http,https +logpath = %(nginx_access_log)s # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year diff --git a/fail2ban/tests/files/logs/nginx-tls-fallback b/fail2ban/tests/files/logs/nginx-tls-fallback new file mode 100644 index 00000000..bd51fbbf --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-tls-fallback @@ -0,0 +1,9 @@ + +# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } +2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T15:47:47", "match": true , "host": "80.191.166.166" } +2020/11/25 15:47:47 [crit] 76952#76952: *5062354 SSL_do_handshake() failed (SSL: error:1408F0A0:SSL routines:ssl3_get_record:length too short) while SSL handshaking, client: 80.191.166.166, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:48:08", "match": true , "host": "5.126.32.148" } +2020/11/25 16:48:08 [crit] 76952#76952: *7976400 SSL_do_handshake() failed (SSL: error:1408F096:SSL routines:ssl3_get_record:encrypted length too long) while SSL handshaking, client: 5.126.32.148, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:02:45", "match": false } +2020/11/25 16:02:45 [error] 76952#76952: *5645766 connect() failed (111: Connection refused) while connecting to upstream, client: 5.126.32.148, server: www.google.de, request: "GET /admin/config HTTP/2.0", upstream: "http://127.0.0.1:3000/admin/config", host: "www.google.de" From c0256724a7b091cba7c774b1774d6e1dc6ee604a Mon Sep 17 00:00:00 2001 From: stepodev Date: Wed, 25 Nov 2020 21:30:21 +0100 Subject: [PATCH 007/139] fix monitoring wrong error log. was access log, should be error.log --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index a7818a06..f7eeeeb8 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -399,7 +399,7 @@ logpath = %(nginx_access_log)s [nginx-tls-fallback] port = http,https -logpath = %(nginx_access_log)s +logpath = %(nginx_error_log)s # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year From d959f6d19926ff2b75322acb008dc692b6a29aa5 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 26 Nov 2020 12:25:32 +0100 Subject: [PATCH 008/139] Update nginx-tls-fallback.conf more precise and conclusive regex without catch-all's --- config/filter.d/nginx-tls-fallback.conf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/filter.d/nginx-tls-fallback.conf b/config/filter.d/nginx-tls-fallback.conf index aaa84fb9..b9eeac04 100644 --- a/config/filter.d/nginx-tls-fallback.conf +++ b/config/filter.d/nginx-tls-fallback.conf @@ -5,12 +5,10 @@ [Definition] -failregex = ^ \[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed.*?ssl3_get_record.*?too.*?, client: , server: \S+$ +failregex = ^\s*\[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: ignoreregex = datepattern = {^LN-BEG} # Author: Stephan Orlowsky -# maybe not restrictive enough, will also match: -#"[crit] 76952#76952: *5062354 SSL_do_handshake() failed ssl3_get_record too, client: 0.0.0.0, server: thisshouldntmatch" From d0ba27cf46411be9104ad5f90e8d39551419fa2a Mon Sep 17 00:00:00 2001 From: stepodev Date: Mon, 30 Nov 2020 12:14:49 +0100 Subject: [PATCH 009/139] move nginx-tls-fallback rules to nginx-http-auth --- ChangeLog | 2 -- config/filter.d/nginx-http-auth.conf | 18 +++++++++++++++++- config/filter.d/nginx-tls-fallback.conf | 14 -------------- config/jail.conf | 4 ---- fail2ban/tests/files/logs/nginx-http-auth | 18 ++++++++++++++++++ fail2ban/tests/files/logs/nginx-tls-fallback | 9 --------- 6 files changed, 35 insertions(+), 30 deletions(-) delete mode 100644 config/filter.d/nginx-tls-fallback.conf delete mode 100644 fail2ban/tests/files/logs/nginx-tls-fallback diff --git a/ChangeLog b/ChangeLog index bd4a369d..f9fb8a33 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,8 +21,6 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) -* `filter.d/nginx-tls-fallback` -- filter added for tls downgrade probes - ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools ----------- diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 93341cd2..d5655fa8 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -3,15 +3,31 @@ [Definition] +mdre-auth = ^ \[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$ +mdre-fallback = ^\s*\[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: -failregex = ^ \[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$ + +mdre-normal = %(mdre-auth)s +mdre-aggressive = %(mdre-auth)s + %(mdre-fallback)s + +failregex = > ignoreregex = datepattern = {^LN-BEG} +mode = normal + # DEV NOTES: +# mdre-auth: # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Extensive search of all nginx auth failures not done yet. # # Author: Daniel Black + +# mdre-fallback: +# Ban people checking for TLS_FALLBACK_SCSV repeatedly +# https://stackoverflow.com/questions/28010492/nginx-critical-error-with-ssl-handshaking/28010608#28010608 +# Author: Stephan Orlowsky + diff --git a/config/filter.d/nginx-tls-fallback.conf b/config/filter.d/nginx-tls-fallback.conf deleted file mode 100644 index b9eeac04..00000000 --- a/config/filter.d/nginx-tls-fallback.conf +++ /dev/null @@ -1,14 +0,0 @@ -# fail2ban filter configuration for nginx -# Ban people checking for TLS_FALLBACK_SCSV repeatedly -# https://stackoverflow.com/questions/28010492/nginx-critical-error-with-ssl-handshaking/28010608#28010608 - -[Definition] - - -failregex = ^\s*\[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: - -ignoreregex = - -datepattern = {^LN-BEG} - -# Author: Stephan Orlowsky diff --git a/config/jail.conf b/config/jail.conf index f7eeeeb8..3d971892 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -397,10 +397,6 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_access_log)s -[nginx-tls-fallback] -port = http,https -logpath = %(nginx_error_log)s - # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year # of usage in production environments. diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth index c9c96807..fb24b242 100644 --- a/fail2ban/tests/files/logs/nginx-http-auth +++ b/fail2ban/tests/files/logs/nginx-http-auth @@ -1,3 +1,4 @@ +# filterOptions: [{"mode": "normal"}, {"mode": "auth"}] # failJSON: { "time": "2012-04-09T11:53:29", "match": true , "host": "192.0.43.10" } 2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" @@ -11,3 +12,20 @@ 2014/04/03 22:20:38 [error] 30708#0: *3 user "scriben dio": password mismatch, client: 192.0.2.1, server: , request: "GET / HTTP/1.1", host: "localhost:8443" # failJSON: { "time": "2014-04-03T22:20:40", "match": true, "host": "192.0.2.2", "desc": "trying injection on user name"} 2014/04/03 22:20:40 [error] 30708#0: *3 user "test": password mismatch, client: 127.0.0.1, server: test, request: "GET / HTTP/1.1", host: "localhost:8443"": was not found in "/etc/nginx/.htpasswd", client: 192.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443" + +# filterOptions: [{"mode": "fallback"}] + +# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } +2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T15:47:47", "match": true , "host": "80.191.166.166" } +2020/11/25 15:47:47 [crit] 76952#76952: *5062354 SSL_do_handshake() failed (SSL: error:1408F0A0:SSL routines:ssl3_get_record:length too short) while SSL handshaking, client: 80.191.166.166, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:48:08", "match": true , "host": "5.126.32.148" } +2020/11/25 16:48:08 [crit] 76952#76952: *7976400 SSL_do_handshake() failed (SSL: error:1408F096:SSL routines:ssl3_get_record:encrypted length too long) while SSL handshaking, client: 5.126.32.148, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:02:45", "match": false } +2020/11/25 16:02:45 [error] 76952#76952: *5645766 connect() failed (111: Connection refused) while connecting to upstream, client: 5.126.32.148, server: www.google.de, request: "GET /admin/config HTTP/2.0", upstream: "http://127.0.0.1:3000/admin/config", host: "www.google.de" + +# filterOptions: [{"mode": "aggressive"}] +# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } +2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 +# failJSON: { "time": "2012-04-09T11:53:29", "match": true , "host": "192.0.43.10" } +2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" diff --git a/fail2ban/tests/files/logs/nginx-tls-fallback b/fail2ban/tests/files/logs/nginx-tls-fallback deleted file mode 100644 index bd51fbbf..00000000 --- a/fail2ban/tests/files/logs/nginx-tls-fallback +++ /dev/null @@ -1,9 +0,0 @@ - -# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } -2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 -# failJSON: { "time": "2020-11-25T15:47:47", "match": true , "host": "80.191.166.166" } -2020/11/25 15:47:47 [crit] 76952#76952: *5062354 SSL_do_handshake() failed (SSL: error:1408F0A0:SSL routines:ssl3_get_record:length too short) while SSL handshaking, client: 80.191.166.166, server: 0.0.0.0:443 -# failJSON: { "time": "2020-11-25T16:48:08", "match": true , "host": "5.126.32.148" } -2020/11/25 16:48:08 [crit] 76952#76952: *7976400 SSL_do_handshake() failed (SSL: error:1408F096:SSL routines:ssl3_get_record:encrypted length too long) while SSL handshaking, client: 5.126.32.148, server: 0.0.0.0:443 -# failJSON: { "time": "2020-11-25T16:02:45", "match": false } -2020/11/25 16:02:45 [error] 76952#76952: *5645766 connect() failed (111: Connection refused) while connecting to upstream, client: 5.126.32.148, server: www.google.de, request: "GET /admin/config HTTP/2.0", upstream: "http://127.0.0.1:3000/admin/config", host: "www.google.de" From cecc3d62ffac65b3545f665dee51eff58c3a7f6e Mon Sep 17 00:00:00 2001 From: stepodev Date: Mon, 30 Nov 2020 12:26:32 +0100 Subject: [PATCH 010/139] add mode explanation to nginx-http-auth in jail.conf --- config/jail.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 3d971892..1b82b648 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -375,8 +375,11 @@ banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log +# To use more aggressive http-auth modes set filter parameter "mode" in jail.local: +# normal (default), aggressive (combines all), auth or fallback +# See "tests/files/logs/nginx-http-auth" or "filter.d/nginx-http-auth.conf" for usage example and details. [nginx-http-auth] - +# mode = normal port = http,https logpath = %(nginx_error_log)s From 747d4683221b5584f9663695fb48145689b42ceb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2021 02:42:38 +0100 Subject: [PATCH 011/139] fixes century selector of %ExY and %Exy in datepattern for tests, considering interval from 2005 (alternate now) to now; + better grouping algorithm for resulting century RE --- fail2ban/server/strptime.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 1464a96d..39fc7958 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -36,10 +36,30 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo Thereby respect possible run in the test-cases (alternate date used there) """ cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) exprset = set( cent(now[0].year + i) for i in (-1, distance) ) if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in (-1, distance) ) - return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset) + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) timeRE = TimeRE() From 0a7f5b7b5c7b352e14c13a8058d0a88753e5f48a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:29:18 +0100 Subject: [PATCH 012/139] CI: add python 3.10 to github actions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a1d31df..c5bb5df2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 3c8aa0e6ba4ca46aa730bbb957653e0468365b49 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:34:08 +0100 Subject: [PATCH 013/139] amend with correct yaml notation (float vs string) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5bb5df2..7d69d2e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 097d9ea7fe4cb2e9285203ca56646d0ea003b064 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:36:09 +0100 Subject: [PATCH 014/139] try wildcard (3.10 is still alpha) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d69d2e5..78f40b8a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.*', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From c9907bef0d6fd4d26c9f7a92611cfd03ea434dca Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:37:34 +0100 Subject: [PATCH 015/139] CI: 3.10.0-alpha.4 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78f40b8a..07086636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.*', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-alpha.4', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 2781e7b9d1111005e2bc97c96ddb53f0fbf47f99 Mon Sep 17 00:00:00 2001 From: Michael Haro Date: Fri, 8 Jan 2021 15:01:25 -0800 Subject: [PATCH 016/139] Add blank line so markdown renders correctly --- FILTERS | 1 + 1 file changed, 1 insertion(+) diff --git a/FILTERS b/FILTERS index e114973a..2ed6281d 100644 --- a/FILTERS +++ b/FILTERS @@ -278,6 +278,7 @@ to tune it. fail2ban-regex -D ... will present Debuggex URLs for the regexs and sample log files that you pass into it. In general use when using regex debuggers for generating fail2ban filters: + * use regex from the ./fail2ban-regex output (to ensure all substitutions are done) * replace with (?&.ipv4) From 9df332fdef853c0ae27578d84beda3a1c41deee1 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 11 Jan 2021 15:10:53 +0100 Subject: [PATCH 017/139] filter.d/apache-overflows.conf: extended to match AH00126 error (Invalid URI ...); closes gh-2908 --- config/filter.d/apache-overflows.conf | 2 +- fail2ban/tests/files/logs/apache-overflows | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 02a2ef20..0f54da11 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,7 +8,7 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) +failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) ignoreregex = diff --git a/fail2ban/tests/files/logs/apache-overflows b/fail2ban/tests/files/logs/apache-overflows index 376114c4..4be013eb 100644 --- a/fail2ban/tests/files/logs/apache-overflows +++ b/fail2ban/tests/files/logs/apache-overflows @@ -3,6 +3,8 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# failJSON: { "time": "2010-03-15T16:04:06", "match": true , "host": "192.0.2.1", "desc": "AH00126 failure, gh-2908" } +[Sat Mar 15 16:04:06.105212 2010] [core:error] [pid 17408] [client 192.0.2.1:55280] AH00126: Invalid URI in request GET /static/../../../a/../../../../etc/passwd HTTP/1.1 # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } From 5f3f4d1e2ffcf99384c96c9800d1aa913ccce81b Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 11 Jan 2021 15:23:40 +0100 Subject: [PATCH 018/139] action.d/cloudflare.conf: better IPv6 capability closes gh-2891 --- config/action.d/cloudflare.conf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 361cb177..4af87080 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -44,7 +44,7 @@ actioncheck = #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ - -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + -d '{"mode":"block","configuration":{"target":"","value":""},"notes":"Fail2Ban "}' \ <_cf_api_url> # Option: actionunban @@ -59,7 +59,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ + "<_cf_api_url>?mode=block&configuration_target=&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" @@ -81,3 +81,8 @@ _cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Conten cftoken = cfuser = + +cftarget = ip + +[Init?family=inet6] +cftarget = ip6 From 164105fab12a8d97050eb1930cbc338b66d6e8eb Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 16 Jan 2021 17:10:12 +0100 Subject: [PATCH 019/139] added new parameter `namespace` for systemd backend closes gh-2910 --- fail2ban/server/filtersystemd.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 47fc891e..1b33b115 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -94,6 +94,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # be sure all journal types will be opened if files specified (don't set flags): if 'files' not in args or not len(args['files']): args['flags'] = 4 + + try: + args['namespace'] = kwargs.pop('namespace') + except KeyError: + pass return args From 0f44a3408aa88d32ab2c4583ac6f54ea82220069 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 19 Jan 2021 17:07:03 +0100 Subject: [PATCH 020/139] amend to 747d4683221b5584f9663695fb48145689b42ceb: fail2ban-regex: loosen up date patterns %ExY, %Exy - let accept every year from 19xx up to current century (+3 years) --- fail2ban/client/fail2banregex.py | 11 +++-- fail2ban/server/datedetector.py | 2 +- fail2ban/server/datetemplate.py | 3 +- fail2ban/server/strptime.py | 81 ++++++++++++++++++-------------- 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index e7a4e214..5d5f4a1c 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -39,7 +39,6 @@ import os import shlex import sys import time -import time import urllib from optparse import OptionParser, Option @@ -52,7 +51,7 @@ except ImportError: from ..version import version, normVersion from .filterreader import FilterReader -from ..server.filter import Filter, FileContainer +from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ @@ -269,15 +268,19 @@ class Fail2banRegex(object): self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.timezone: self._filter.setLogTimeZone(opts.timezone) + self._filter.checkFindTime = False + if True: # not opts.out: + MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns) + from ..server.strptime import _updateTimeRE + _updateTimeRE() if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: self._filter.setUseDns(opts.usedns) self._filter.returnRawHost = opts.raw - self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) - self._filter.ignorePending = opts.out + self._filter.ignorePending = bool(opts.out) # callback to increment ignored RE's by index (during process): self._filter.onIgnoreRegex = self._onIgnoreRegex self._backend = 'auto' diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 90a70b0d..ecc9d935 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -35,7 +35,7 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -logLevel = 6 +logLevel = 5 RE_DATE_PREMATCH = re.compile(r"(? 1: - for i in exprset: - if c is None or i[0:-1] == c: - c = i[0:-1] - else: - c = None - break - if not c: - for i in exprset: - if c is None or i[0] == c: - c = i[0] - else: - c = None - break - if c: - return "%s%s" % (c, grp([i[len(c):] for i in exprset])) - return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ - if len(exprset) > 1 else "".join(exprset) - exprset = set( cent(now[0].year + i) for i in (-1, distance) ) - if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) - return grp(sorted(list(exprset))) - timeRE = TimeRE() # %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, @@ -92,11 +61,51 @@ timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" -# more precise year patterns, within same century of last year and -# the next 3 years (for possible long uptime of fail2ban); thereby -# respect possible run in the test-cases (alternate date used there): -timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) -timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) + +def _updateTimeRE(): + def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): + """ Build century regex for last year and the next years (distance). + + Thereby respect possible run in the test-cases (alternate date used there) + """ + cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) + exprset = set( cent(now[0].year + i) for i in (-1, distance) ) + if len(now) > 1 and now[1]: + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) + + # more precise year patterns, within same century of last year and + # the next 3 years (for possible long uptime of fail2ban); thereby + # respect possible run in the test-cases (alternate date used there): + if MyTime.alternateNowTime != 0: + timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) + timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) + else: # accept years: 19xx|2xxx up to current century + timeRE['ExY'] = r"(?P(?:19\d{2}|%s\d))" % _getYearCentRE(cent=(0,3), distance=3, + now=(MyTime.now(), datetime.datetime.fromtimestamp(978393600))) + timeRE['Exy'] = r"(?P\d{2})" + +_updateTimeRE() def getTimePatternRE(): keys = timeRE.keys() From 913c37db80d3f78c37e7df2f18a503baf3f6dabc Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 21 Jan 2021 18:56:25 +0100 Subject: [PATCH 021/139] more fixes and optimizations, better RE's for patterns, allow parse date without time with such a datepattern (assume 00:00:00 then), etc --- fail2ban/server/strptime.py | 22 +++++++++++++++------- fail2ban/tests/datedetectortestcase.py | 3 +++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 83cb3989..69514b20 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -52,15 +52,23 @@ timeRE['z'] = r"(?PZ|UTC|GMT|[+-][01]\d(?::?\d{2})?)" timeRE['ExZ'] = r"(?P%s)" % (TZ_ABBR_RE,) timeRE['Exz'] = r"(?P(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE) +# overwrite default patterns, since they can be non-optimal: +timeRE['d'] = r"(?P[1-2]\d|[0 ]?[1-9]|3[0-1])" +timeRE['m'] = r"(?P0?[1-9]|1[0-2])" +timeRE['Y'] = r"(?P\d{4})" +timeRE['H'] = r"(?P[0-1]?\d|2[0-3])" +timeRE['M'] = r"(?P[0-5]?\d)" +timeRE['S'] = r"(?P[0-5]?\d|6[0-1])" + # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: -timeRE['Exd'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" -timeRE['Exm'] = r"(?P1[0-2]|0[1-9])" -timeRE['ExH'] = r"(?P2[0-3]|[0-1]\d)" -timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" +timeRE['Exd'] = r"(?P[1-2]\d|0[1-9]|3[0-1])" +timeRE['Exm'] = r"(?P0[1-9]|1[0-2])" +timeRE['ExH'] = r"(?P[0-1]\d|2[0-3])" +timeRE['Exk'] = r" ?(?P[0-1]?\d|2[0-3])" timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" -timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" +timeRE['ExS'] = r"(?P[0-5]\d|6[0-1])" def _updateTimeRE(): def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): @@ -197,9 +205,9 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """ now = \ - year = month = day = hour = minute = tzoffset = \ + year = month = day = tzoffset = \ weekday = julian = week_of_year = None - second = fraction = 0 + hour = minute = second = fraction = 0 for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index d6370fc4..b8e8451e 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -551,6 +551,9 @@ class CustomDateFormatsTest(unittest.TestCase): (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), + # no time (only date) in pattern, assume local 00:00:00 for H:M:S : + (1093989600.0, "^%Y-%m-%d**", '2004-09-01'), + (1093996800.0, "^%Y-%m-%d%z**", '2004-09-01Z'), ): logSys.debug('== test: %r', (matched, dp, line)) dd = DateDetector() From 3700a9e5230822d33b2377afed9b6de38676e564 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 26 Jan 2021 20:25:58 +0100 Subject: [PATCH 022/139] invalidate IP/DNS caches by reload, so inter alia would allow to recognize IPv6IsAllowed immediately, previously retarded up to cache max-time (5m); closes gh-2804 --- fail2ban/server/server.py | 9 ++++++++- fail2ban/tests/utils.py | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index b12c8f9f..475fd706 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -33,7 +33,7 @@ import stat import sys from .jails import Jails -from .filter import FileFilter, JournalFilter +from .filter import DNSUtils, FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version @@ -274,6 +274,11 @@ class Server: for name in self.__jails.keys(): self.delJail(name, stop=False, join=True) + def clearCaches(self): + # we need to clear caches, to be able to recognize new IPs/families etc: + DNSUtils.CACHE_nameToIp.clear() + DNSUtils.CACHE_ipToName.clear() + def reloadJails(self, name, opts, begin): if begin: # begin reload: @@ -295,6 +300,8 @@ class Server: if "--restart" in opts: self.stopJail(name) else: + # invalidate caches by reload + self.clearCaches() # first unban all ips (will be not restored after (re)start): if "--unban" in opts: self.setUnbanIP() diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b54581f5..921427db 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -320,6 +320,7 @@ def initTests(opts): # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName + c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) for i in xrange(256): @@ -337,6 +338,7 @@ def initTests(opts): c.set('8.8.4.4', 'dns.google') # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp + c.clear = lambda: logSys.warn('clear CACHE_nameToIp is disabled in test suite') for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), From c75748c5d3b4de1e51318e1db4a5b72519c47c1f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 27 Jan 2021 17:05:37 +0100 Subject: [PATCH 023/139] fail2ban.conf: added new fail2ban configuration option "allowipv6" (default auto), can be used to allow or disallow IPv6 interface in fail2ban immediately by start (e. g. if fail2ban starts before network interfaces). closes gh-2804 --- config/fail2ban.conf | 6 ++++ fail2ban/client/fail2banreader.py | 2 ++ fail2ban/server/ipdns.py | 41 +++++++++++++++++++------- fail2ban/server/server.py | 5 ++++ fail2ban/server/transmitter.py | 5 ++++ fail2ban/tests/clientreadertestcase.py | 1 + fail2ban/tests/servertestcase.py | 15 +++++++++- man/jail.conf.5 | 5 ++++ 8 files changed, 69 insertions(+), 11 deletions(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index f3867839..601402d8 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -55,6 +55,12 @@ socket = /var/run/fail2ban/fail2ban.sock # pidfile = /var/run/fail2ban/fail2ban.pid +# Option: allowipv6 +# Notes.: Allows IPv6 interface: +# Default: auto +# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto +#allowipv6 = auto + # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. # A value of ":memory:" means database is only stored in memory diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 3270b767..1f135cf8 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -53,6 +53,7 @@ class Fail2banReader(ConfigReader): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], + ["string", "allowipv6", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbmaxmatches", None], ["string", "dbpurgeage", "1d"]] @@ -74,6 +75,7 @@ class Fail2banReader(ConfigReader): # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, + "allowipv6": 14, "dbfile":50, "dbmaxmatches":51, "dbpurgeage":51} stream = list() for opt in self.__opts: diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 571ccc4f..5f3e4571 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -169,27 +169,31 @@ class DNSUtils: DNSUtils.CACHE_ipToName.set(key, name) return name + # key find cached own hostnames (this tuple-key cannot be used elsewhere): + _getSelfNames_key = ('self','dns') + @staticmethod def getSelfNames(): """Get own host names of self""" - # try find cached own hostnames (this tuple-key cannot be used elsewhere): - key = ('self','dns') - names = DNSUtils.CACHE_ipToName.get(key) + # try find cached own hostnames: + names = DNSUtils.CACHE_ipToName.get(DNSUtils._getSelfNames_key) # get it using different ways (a set with names of localhost, hostname, fully qualified): if names is None: names = set([ 'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True) ]) - set(['']) # getHostname can return '' # cache and return : - DNSUtils.CACHE_ipToName.set(key, names) + DNSUtils.CACHE_ipToName.set(DNSUtils._getSelfNames_key, names) return names + # key to find cached own IPs (this tuple-key cannot be used elsewhere): + _getSelfIPs_key = ('self','ips') + @staticmethod def getSelfIPs(): """Get own IP addresses of self""" - # try find cached own IPs (this tuple-key cannot be used elsewhere): - key = ('self','ips') - ips = DNSUtils.CACHE_nameToIp.get(key) + # to find cached own IPs: + ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getSelfIPs_key) # get it using different ways (a set with IPs of localhost, hostname, fully qualified): if ips is None: ips = set() @@ -199,13 +203,30 @@ class DNSUtils: except Exception as e: # pragma: no cover logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) # cache and return : - DNSUtils.CACHE_nameToIp.set(key, ips) + DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips) return ips + _IPv6IsAllowed = None + + @staticmethod + def setIPv6IsAllowed(value): + DNSUtils._IPv6IsAllowed = value + logSys.debug("IPv6 is %s", ('on' if value else 'off') if value is not None else 'auto') + return value + + # key to find cached value of IPv6 allowance (this tuple-key cannot be used elsewhere): + _IPv6IsAllowed_key = ('self','ipv6-allowed') + @staticmethod def IPv6IsAllowed(): - # return os.path.exists("/proc/net/if_inet6") || any((':' in ip) for ip in DNSUtils.getSelfIPs()) - return any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + if DNSUtils._IPv6IsAllowed is not None: + return DNSUtils._IPv6IsAllowed + v = DNSUtils.CACHE_nameToIp.get(DNSUtils._IPv6IsAllowed_key) + if v is not None: + return v + v = any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + DNSUtils.CACHE_nameToIp.set(DNSUtils._IPv6IsAllowed_key, v) + return v ## diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 475fd706..bd2c7ad3 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -770,6 +770,11 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + @staticmethod + def setIPv6IsAllowed(value): + value = _as_bool(value) if value != 'auto' else None + return DNSUtils.setIPv6IsAllowed(value) + def setThreadOptions(self, value): for o, v in value.iteritems(): if o == 'stacksize': diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 10cfd163..e40f2f51 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -173,6 +173,11 @@ class Transmitter: return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + elif name == "allowipv6": + value = command[1] + self.__server.setIPv6IsAllowed(value) + if self.__quiet: return + return value #Thread elif name == "thread": value = command[1] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 2cfaff77..54850bca 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -975,6 +975,7 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'syslogsocket', 'auto'], ['set', 'loglevel', "INFO"], ['set', 'logtarget', '/var/log/fail2ban.log'], + ['set', 'allowipv6', 'auto'], ['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'], ['set', 'dbmaxmatches', 10], ['set', 'dbpurgeage', '1d'], diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index eaf1b346..b7b9d802 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -35,7 +35,7 @@ import platform from ..server.failregex import Regex, FailRegex, RegexException from ..server import actions as _actions from ..server.server import Server -from ..server.ipdns import IPAddr +from ..server.ipdns import DNSUtils, IPAddr from ..server.jail import Jail from ..server.jailthread import JailThread from ..server.ticket import BanTicket @@ -175,6 +175,19 @@ class Transmitter(TransmitterBase): def testVersion(self): self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) + def testSetIPv6(self): + try: + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'yes']), (0, 'yes')) + self.assertTrue(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is on"); self.pruneLog() + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'no']), (0, 'no')) + self.assertFalse(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is off"); self.pruneLog() + finally: + # restore back to auto: + self.assertEqual(self.transm.proceed(["set", "allowipv6", "auto"]), (0, "auto")) + self.assertLogged("IPv6 is auto"); self.pruneLog() + def testSleep(self): if not unittest.F2B.fast: t0 = time.time() diff --git a/man/jail.conf.5 b/man/jail.conf.5 index dc226ac2..788fad2b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -151,6 +151,11 @@ PID filename. Default: /var/run/fail2ban/fail2ban.pid .br This is used to store the process ID of the fail2ban server. .TP +.B allowipv6 +option to allow IPv6 interface - auto, yes (on, true, 1) or no (off, false, 0). Default: auto +.br +This value can be used to declare fail2ban whether IPv6 is allowed or not. +.TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 .br From dc4ee5aa47e211bb8ca7417c73b61d4c3dc44a14 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 18 Jan 2021 17:43:06 -0500 Subject: [PATCH 024/139] Add transport to asterisk RE Call rejection messages from Asterisk can have the transport prefixed to the IP address. Signed-off-by: Brian J. Murrell --- config/filter.d/asterisk.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 68472495..e15d7bfe 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -21,7 +21,7 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])?:? [^:]+ prefregex = ^%(__prefix_line)s%(log_prefix)s .+$ failregex = ^Registration from '[^']*' failed for '(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context + ^Call from '[^']*' \((?:(?:TCP|UDP):)?:\d+\) to extension '[^']*' rejected because extension not found in context ^(?:Host )? (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b) ^No registration for peer '[^']*' \(from \)$ ^hacking attempt detected ''$ From 69c96c00c0ed6cdeb2e90a496e91b569c14cef14 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 18 Jan 2021 17:51:38 -0500 Subject: [PATCH 025/139] Log entries for updated failregex Add a sample failregex. Signed-off-by: Brian J. Murrell --- fail2ban/tests/files/logs/asterisk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 76ec40b2..0c4f81bf 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -19,6 +19,8 @@ [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. +# failJSON: { "time": "Jan 18 17:39:50", "match": true , "host": "1.2.3.4" } +[Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'. # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found # failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" } From 7f185a828e6573669535b152fb945a366270d809 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 19 Jan 2021 08:42:05 -0500 Subject: [PATCH 026/139] Update date in failJSON The date format in failJSON is specific, so convert the date to use that format. --- fail2ban/tests/files/logs/asterisk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 0c4f81bf..ab31fa6f 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -19,7 +19,7 @@ [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. -# failJSON: { "time": "Jan 18 17:39:50", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-01-18T17:39:50", "match": true , "host": "1.2.3.4" } [Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'. # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found From 366c64cb9da3c5d833eefc83f5bb200713db29b5 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Feb 2021 14:45:30 +0100 Subject: [PATCH 027/139] extractOptions: ensure options are parsed completely - avoids unexpected skip or truncate of parameters, produces more verbose error message in case of incorrect syntax; added more tests covering several cases WARN: potential incompatibility (since it doesn't silently ignore wrong syntax anymore) --- fail2ban/client/fail2banregex.py | 40 +++++++++++++++---------- fail2ban/client/jailreader.py | 15 +++++----- fail2ban/helpers.py | 12 ++++++-- fail2ban/tests/clientreadertestcase.py | 17 ++++++----- fail2ban/tests/fail2banregextestcase.py | 6 ++++ 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 5d5f4a1c..90e178f9 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -35,6 +35,7 @@ __license__ = "GPL" import getopt import logging +import re import os import shlex import sys @@ -329,26 +330,33 @@ class Fail2banRegex(object): regex = regextype + 'regex' # try to check - we've case filter?[options...]?: basedir = self._opts.config + fltName = value fltFile = None fltOpt = {} if regextype == 'fail': - fltName, fltOpt = extractOptions(value) - if fltName is not None: - if "." in fltName[~5:]: - tryNames = (fltName,) - else: - tryNames = (fltName, fltName + '.conf', fltName + '.local') - for fltFile in tryNames: - if not "/" in fltFile: - if os.path.basename(basedir) == 'filter.d': - fltFile = os.path.join(basedir, fltFile) - else: - fltFile = os.path.join(basedir, 'filter.d', fltFile) + if re.search(r'^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): + try: + fltName, fltOpt = extractOptions(value) + if "." in fltName[~5:]: + tryNames = (fltName,) else: - basedir = os.path.dirname(fltFile) - if os.path.isfile(fltFile): - break - fltFile = None + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + except Exception as e: + output("ERROR: Wrong filter name or options: %s" % (str(e),)) + output(" while parsing: %s" % (value,)) + if self._verbose: raise(e) + return False # if it is filter file: if fltFile is not None: if (basedir == self._opts.config diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 1d7db0dc..0a27c644 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -133,9 +133,10 @@ class JailReader(ConfigReader): # Read filter flt = self.__opts["filter"] if flt: - filterName, filterOpt = extractOptions(flt) - if not filterName: - raise JailDefError("Invalid filter definition %r" % flt) + try: + filterName, filterOpt = extractOptions(flt) + except ValueError as e: + raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) @@ -167,10 +168,10 @@ class JailReader(ConfigReader): if not act: # skip empty actions continue # join with previous line if needed (consider possible new-line): - actName, actOpt = extractOptions(act) - prevln = '' - if not actName: - raise JailDefError("Invalid action definition %r" % act) + try: + actName, actOpt = extractOptions(act) + except ValueError as e: + raise JailDefError("Invalid action definition %r: %s" % (act, e)) if actName.endswith(".py"): self.__actions.append([ "set", diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index c45be849..5c1750a6 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -371,7 +371,7 @@ OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL) # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` OPTION_EXTRACT_CRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) + r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P.+))|,?\s*$|(?P.+)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) @@ -379,13 +379,19 @@ OPTION_SPLIT_CRE = re.compile( def extractOptions(option): match = OPTION_CRE.match(option) if not match: - # TODO proper error handling - return None, None + raise ValueError("unexpected option syntax") option_name, optstr = match.groups() option_opts = dict() if optstr: for optmatch in OPTION_EXTRACT_CRE.finditer(optstr): + if optmatch.group("wrngA"): + raise ValueError("unexpected syntax at %d after option %r: %s" % ( + optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25])) + if optmatch.group("wrngB"): + raise ValueError("expected option, wrong syntax at %d: %s" % ( + optmatch.start("wrngB"), optmatch.group("wrngB")[0:25])) opt = optmatch.group(1) + if not opt: continue value = [ val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 54850bca..e92edd48 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -381,13 +381,16 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]")) self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is")) - self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']")) - #print(self.getLog()) - #self.assertLogged("Invalid argument ['s'] in ''s''") - self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']")) + self.assertEqual(('mail', {'a': 'b'}), extractOptions("mail[a=b, ]")) - #self.assertRaises(ValueError, extractOptions ,'mail-how[') + self.assertRaises(ValueError, extractOptions ,'mail-how[') + + self.assertRaises(ValueError, extractOptions, """mail[a="test with interim (wrong) "" quotes"]""") + self.assertRaises(ValueError, extractOptions, """mail[a='test with interim (wrong) '' quotes']""") + self.assertRaises(ValueError, extractOptions, """mail[a='x, y, z', b=x, y, z]""") + + self.assertRaises(ValueError, extractOptions, """mail['s']""") # Empty option option = "abc[]" @@ -752,9 +755,9 @@ class JailsReaderTest(LogCaptureTestCase): ['add', 'tz_correct', 'auto'], ['start', 'tz_correct'], ['config-error', - "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], + "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"], ['config-error', - "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"], + "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"], ['config-error', "Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"], ['config-error', diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 4d878a24..85fe4f15 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -141,6 +141,12 @@ class Fail2banRegexTest(LogCaptureTestCase): )) self.assertLogged("Unable to compile regular expression") + def testWrongFilterOptions(self): + self.assertFalse(_test_exec( + "test", "flt[a='x,y,z',b=z,y,x]" + )) + self.assertLogged("Wrong filter name or options", "wrong syntax at 14: y,x", all=True) + def testDirectFound(self): self.assertTrue(_test_exec( "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", From d678440658541a3c02c8d57695bd363f72147724 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 11 Feb 2021 18:32:32 +0100 Subject: [PATCH 028/139] more precise RE (avoids weakness with catch-all's and is injection safe) --- config/filter.d/drupal-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index 2d4cbe9f..2404cc6d 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.*\|\d\|.*\|Login attempt failed (?:for|from) .+\.$ +failregex = ^%(__prefix_line)s(?:https?:\/\/)[^|]+\|[^|]+\|[^|]+\|\|(?:[^|]*\|)*Login attempt failed (?:for|from) [^|]+\.$ ignoreregex = From f4f92aa72d4124fbdff615e14fc5bfb8d66d7b0a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 11 Feb 2021 18:56:53 +0100 Subject: [PATCH 029/139] more tests covering different cases, injections attempt etc --- fail2ban/tests/files/logs/drupal-auth | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/drupal-auth b/fail2ban/tests/files/logs/drupal-auth index 5e7194d9..4d063e55 100644 --- a/fail2ban/tests/files/logs/drupal-auth +++ b/fail2ban/tests/files/logs/drupal-auth @@ -3,5 +3,15 @@ Apr 26 13:15:25 webserver example.com: https://example.com|1430068525|user|1.2.3 # failJSON: { "time": "2005-04-26T13:15:25", "match": true , "host": "1.2.3.4" } Apr 26 13:15:25 webserver example.com: https://example.com/subdir|1430068525|user|1.2.3.4|https://example.com/subdir/user|https://example.com/subdir/user|0||Login attempt failed for drupaladmin. -# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4" } +# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4", "user": "drupaladmin" } Apr 26 13:19:08 webserver example.com: https://example.com|1430068748|user|1.2.3.4|https://example.com/user|https://example.com/user|1||Session opened for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:00", "match": false, "desc": "attempt to inject on URI (pipe, login failed for), not a failure, gh-2742" } +Apr 26 13:20:00 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed for tester|2||Session revisited for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:01", "match": true , "host": "192.0.2.7", "user": "Jack Sparrow", "desc": "log-format change - for -> from, user name with space, gh-2742" } +Apr 26 13:20:01 mweb drupal_site[24864]: https://www.example.com|1613058599|user|192.0.2.7|https://www.example.com/en/user/login|https://www.example.com/en/user/login|0||Login attempt failed from Jack Sparrow. +# failJSON: { "time": "2005-04-26T13:20:02", "match": true , "host": "192.0.2.4", "desc": "attempt to inject on URI (pipe), login failed, gh-2742" } +Apr 26 13:20:02 host drupal-site: https://example.com|1613063581|user|192.0.2.4|https://example.com/user/login?test=%7C&test2=%7C|https://example.com/user/login?test=|&test2=||0||Login attempt failed from 192.0.2.4. +# failJSON: { "time": "2005-04-26T13:20:03", "match": false, "desc": "attempt to inject on URI (pipe, login failed from), not a failure, gh-2742" } +Apr 26 13:20:03 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed from 1.2.3.4|2||Session revisited for drupaladmin. From 8ae9208454e426aa87b96ba5df26036c4ae5cefd Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 16:44:27 +0100 Subject: [PATCH 030/139] try to provide coverage for 3.10-alpha.5 (#2931) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a1d31df..262448c2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-alpha.5', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 2b6bb2c1bed8f7009631e8f8c306fa3160324a49 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:19:24 +0100 Subject: [PATCH 031/139] follow bpo-37324: :ref:`collections-abstract-base-classes` moved to the :mod:`collections.abc` module (since 3.10-alpha.5 `MutableMapping` is missing in collections module) --- fail2ban/server/action.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 3bc48fe0..f0f1e6f5 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -30,7 +30,10 @@ import tempfile import threading import time from abc import ABCMeta -from collections import MutableMapping +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from .failregex import mapTag2Opt from .ipdns import DNSUtils From 42dee38ad2ac5c3f23bdf297d824022923270dd9 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:25:45 +0100 Subject: [PATCH 032/139] amend for `Mapping` --- fail2ban/server/actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index b7b95b44..897d907c 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -28,7 +28,10 @@ import logging import os import sys import time -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping try: from collections import OrderedDict except ImportError: From 9f1d1f4fbd0804695a976beb191f2c49a2739834 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:35:59 +0100 Subject: [PATCH 033/139] amend for `Mapping` (jails) --- fail2ban/server/jails.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py index 972a8c4b..27e12ddf 100644 --- a/fail2ban/server/jails.py +++ b/fail2ban/server/jails.py @@ -22,7 +22,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" from threading import Lock -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from ..exceptions import DuplicateJailException, UnknownJailException from .jail import Jail From abc5a4e062417139893ba1f402e09e96f8887479 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 Feb 2021 19:02:22 +0100 Subject: [PATCH 034/139] ChangeLog (#2742) --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 6c530871..e5e7b485 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition different from 0) in case of unsane environment. ### Fixes +* `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) ### New Features and Enhancements * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair From 55d7d9e214f72bbe4f39a2d17aa004d80bfc7299 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Feb 2021 18:39:58 +0100 Subject: [PATCH 035/139] *WiP* try to solve RC on jails with too many failures without ban, gh-2945 ... --- fail2ban/server/failmanager.py | 5 +++-- fail2ban/server/filter.py | 34 +++++++++++++++++++++++------- fail2ban/server/filtergamin.py | 16 +++----------- fail2ban/server/filterpoll.py | 10 ++------- fail2ban/server/filterpyinotify.py | 15 ++++++------- fail2ban/server/filtersystemd.py | 13 ++++++------ fail2ban/tests/filtertestcase.py | 14 ++++++------ 7 files changed, 53 insertions(+), 54 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 4173a233..64576dbd 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -124,9 +124,10 @@ class FailManager: return len(self.__failList) def cleanup(self, time): + time -= self.__maxTime with self.__lock: todelete = [fid for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime <= time] + if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: self.__failList = dict() @@ -140,7 +141,7 @@ class FailManager: else: # create new dictionary without items to be deleted: self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime > time) + if item.getTime() > time) self.__bgSvc.service() def delFailure(self, fid): diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 4e947d27..0f4e9e5b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -93,6 +93,8 @@ class Filter(JailThread): ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## Next service (cleanup) time + self.__nextSvcTime = -(1<<63) ## if set, treat log lines without explicit time zone to be in this time zone self.__logtimezone = None ## Default or preferred encoding (to decode bytes from file or journal): @@ -114,10 +116,10 @@ class Filter(JailThread): self.checkFindTime = True ## shows that filter is in operation mode (processing new messages): self.inOperation = True - ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): - self.banASAP = True ## Ticks counter self.ticks = 0 + ## Processed lines counter + self.procLines = 0 ## Thread name: self.name="f2b/f."+self.jailName @@ -441,12 +443,23 @@ class Filter(JailThread): def performBan(self, ip=None): """Performs a ban for IPs (or given ip) that are reached maxretry of the jail.""" - try: # pragma: no branch - exception is the only way out - while True: + while True: + try: ticket = self.failManager.toBan(ip) - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) + except FailManagerEmpty: + break + self.jail.putFailTicket(ticket) + if ip: break + self.performSvc() + + def performSvc(self, force=False): + """Performs a service tasks (clean failure list).""" + tm = MyTime.time() + # avoid too early clean up: + if force or tm >= self.__nextSvcTime: + self.__nextSvcTime = tm + 5 + # clean up failure list: + self.failManager.cleanup(tm) def addAttempt(self, ip, *matches): """Generate a failed attempt for ip""" @@ -694,8 +707,12 @@ class Filter(JailThread): attempts = self.failManager.addFailure(tick) # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, # we can speedup ban, so do it as soon as possible: - if self.banASAP and attempts >= self.failManager.getMaxRetry(): + if attempts >= self.failManager.getMaxRetry(): self.performBan(ip) + self.procLines += 1 + # every 100 lines check need to perform service tasks: + if self.procLines % 100 == 0: + self.performSvc() # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 @@ -1064,6 +1081,7 @@ class FileFilter(Filter): # is created and is added to the FailManager. def getFailures(self, filename, inOperation=None): + if self.idle: return False log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 078246de..c5373445 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -55,7 +55,6 @@ class FilterGamin(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Gamin monitor self.monitor = gamin.WatchMonitor() fd = self.monitor.get_fd() @@ -67,21 +66,9 @@ class FilterGamin(FileFilter): logSys.log(4, "Got event: " + repr(event) + " for " + path) if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists): logSys.debug("File changed: " + path) - self.__modified = True self.ticks += 1 - self._process_file(path) - - def _process_file(self, path): - """Process a given file - - TODO -- RF: - this is a common logic and must be shared/provided by FileFilter - """ self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False ## # Add a log file path @@ -128,6 +115,9 @@ class FilterGamin(FileFilter): Utils.wait_for(lambda: not self.active or self._handleEvents(), self.sleeptime) self.ticks += 1 + if self.ticks % 10 == 0: + self.performSvc() + logSys.debug("[%s] filter terminated", self.jailName) return True diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 7bbdfc5c..7ee00540 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -27,9 +27,7 @@ __license__ = "GPL" import os import time -from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime from .utils import Utils from ..helpers import getLogger, logging @@ -55,7 +53,6 @@ class FilterPoll(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() @@ -115,13 +112,10 @@ class FilterPoll(FileFilter): break for filename in modlst: self.getFailures(filename) - self.__modified = True self.ticks += 1 - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 9796e26f..d62348a2 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -75,7 +75,6 @@ class FilterPyinotify(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__notifier = None @@ -140,9 +139,6 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -352,9 +348,14 @@ class FilterPyinotify(FileFilter): if not self.active: break self.__notifier.read_events() + self.ticks += 1 + # check pending files/dirs (logrotate ready): - if not self.idle: - self._checkPending() + if self.idle: + continue + self._checkPending() + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -364,8 +365,6 @@ class FilterPyinotify(FileFilter): # incr common error counter: self.commonError() - self.ticks += 1 - logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 1b33b115..925109d1 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -322,13 +322,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover break else: break - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = 0 - # update position in log (time and iso string): - if self.jail.database is not None: - self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__modified = 0 + if self.ticks % 10 == 0: + self.performSvc() + # update position in log (time and iso string): + if self.jail.database is not None: + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2dac91d1..15882ea0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -800,7 +800,6 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -952,15 +951,18 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') + print('=========='*10) self.filter.getFailures(self.name) + print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... + print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) - _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) + #_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1018,7 +1020,6 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1277,14 +1278,14 @@ def get_monitor_failures_testcase(Filter_): # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) # we should detect the failures - self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 12) + self.assertEqual(self.filter.failManager.getFailTotal(), 9) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1316,7 +1317,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1570,7 +1570,6 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1771,7 +1770,6 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) - filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures From e353fb802442309d0b6fbfe86cf6d0ab286c6626 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 02:46:44 +0100 Subject: [PATCH 036/139] fixed test cases (ban ASAP also followed in test suite now, so failure reached maxretry causes immediate ban now) --- fail2ban/tests/filtertestcase.py | 120 +++++++++++++++++-------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 15882ea0..fe37ea29 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -164,18 +164,25 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): # get fail ticket from jail found.append(_ticket_tuple(filter_.getFailTicket())) else: - # when we are testing without jails - # wait for failures (up to max time) - Utils.wait_for( - lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), - _maxWaitTime(10)) - # get fail ticket(s) from filter - while tickcount: - try: - found.append(_ticket_tuple(filter_.failManager.toBan())) - except FailManagerEmpty: - break - tickcount -= 1 + # when we are testing without jails wait for failures (up to max time) + if filter_.jail: + while True: + t = filter_.jail.getFailTicket() + if not t: break + found.append(_ticket_tuple(t)) + if found: + tickcount -= len(found) + if tickcount > 0: + Utils.wait_for( + lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), + _maxWaitTime(10)) + # get fail ticket(s) from filter + while tickcount: + try: + found.append(_ticket_tuple(filter_.failManager.toBan())) + except FailManagerEmpty: + break + tickcount -= 1 if not isinstance(output[0], (tuple,list)): utest.assertEqual(len(found), 1) @@ -951,14 +958,11 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') - print('=========='*10) self.filter.getFailures(self.name) - print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... - print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) @@ -1112,12 +1116,13 @@ def get_monitor_failures_testcase(Filter_): skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def _wait4failures(self, count=2): + def _wait4failures(self, count=2, waitEmpty=True): # Poll might need more time - self.assertTrue(self.isEmpty(_maxWaitTime(5)), - "Queue must be empty but it is not: %s." - % (', '.join([str(x) for x in self.jail.queue]))) - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + if waitEmpty: + self.assertTrue(self.isEmpty(_maxWaitTime(5)), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), count) @@ -1283,9 +1288,9 @@ def get_monitor_failures_testcase(Filter_): # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... - self.assert_correct_last_attempt(GetFailures.FAILURES_01) + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 9) + self._wait4failures(12, False) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1640,6 +1645,7 @@ class GetFailures(LogCaptureTestCase): [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) + self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) self.filter.addFailRegex(r"Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1648,6 +1654,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03(self): output = ('203.162.223.135', 6, 1124013600.0) + self.filter.setMaxRetry(6) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) @@ -1656,6 +1663,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_InOperation(self): output = ('203.162.223.135', 9, 1124013600.0) + self.filter.setMaxRetry(9) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) @@ -1673,7 +1681,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... output = ('203.162.223.135', 2, 1124013600.0) - self.filter.setMaxRetry(1) + self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1683,10 +1691,12 @@ class GetFailures(LogCaptureTestCase): def testGetFailures04(self): # because of not exact time in testcase04.log (no year), we should always use our test time: self.assertEqual(MyTime.time(), 1124013600) - # should find exact 4 failures for *.186 and 2 failures for *.185 - output = (('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 2, 1124013598.0)) - + # should find exact 4 failures for *.186 and 2 failures for *.185, but maxretry is 2, so 3 tickets: + output = ( + ('212.41.96.186', 2, 1124013480.0), + ('212.41.96.186', 2, 1124013600.0), + ('212.41.96.185', 2, 1124013598.0) + ) # speedup search using exact date pattern: self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', @@ -1743,9 +1753,11 @@ class GetFailures(LogCaptureTestCase): unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ( - ('93.184.216.34', 2, 1124013539.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ('93.184.216.34', 1, 1124013299.0, + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ), + ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] @@ -1779,8 +1791,11 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, filter_, output) def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = [ + ('141.3.81.106', 8, 1124013541.0) + ] + self.filter.setMaxRetry(8) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex(r"Failed .* from ") self.filter.addFailRegex(r"Accepted .* from ") @@ -1798,26 +1813,25 @@ class GetFailures(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + + _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1826,14 +1840,17 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + ("192.0.43.15", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1842,14 +1859,9 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) class DNSUtilsTests(unittest.TestCase): From 92a224217496fe3114fc7ee9f80708c00804ec03 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 15:54:48 +0100 Subject: [PATCH 037/139] amend fixing journal tests (systemd backend only) --- fail2ban/tests/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fe37ea29..b9b7e8aa 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1517,7 +1517,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", "TEST_UUID=%s" % self.test_uuid]) - self.assert_correct_ban("193.168.0.128", 4) + self.assert_correct_ban("193.168.0.128", 3) _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures @@ -1531,7 +1531,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=15, n=4) self.waitForTicks(1) self.assertTrue(self.isFilled(10)) - self.assert_correct_ban("87.142.124.10", 4) + self.assert_correct_ban("87.142.124.10", 3) # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", From 6f4b6ec8ccdb68c75aec8225d8fa2b03ed19f320 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2021 13:05:04 +0100 Subject: [PATCH 038/139] action.d/badips.* removed (badips.com is no longer active, gh-2889) --- MANIFEST | 3 - config/action.d/badips.conf | 19 -- config/action.d/badips.py | 391 ------------------------- config/jail.conf | 14 - fail2ban/tests/action_d/test_badips.py | 157 ---------- fail2ban/tests/clientreadertestcase.py | 10 +- 6 files changed, 3 insertions(+), 591 deletions(-) delete mode 100644 config/action.d/badips.conf delete mode 100644 config/action.d/badips.py delete mode 100644 fail2ban/tests/action_d/test_badips.py diff --git a/MANIFEST b/MANIFEST index 50f308db..efe87085 100644 --- a/MANIFEST +++ b/MANIFEST @@ -5,8 +5,6 @@ bin/fail2ban-testcases ChangeLog config/action.d/abuseipdb.conf config/action.d/apf.conf -config/action.d/badips.conf -config/action.d/badips.py config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf @@ -219,7 +217,6 @@ fail2ban/setup.py fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py -fail2ban/tests/action_d/test_badips.py fail2ban/tests/action_d/test_smtp.py fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf deleted file mode 100644 index 6f9513f6..00000000 --- a/config/action.d/badips.conf +++ /dev/null @@ -1,19 +0,0 @@ -# Fail2ban reporting to badips.com -# -# Note: This reports an IP only and does not actually ban traffic. Use -# another action in the same jail if you want bans to occur. -# -# Set the category to the appropriate value before use. -# -# To get see register and optional key to get personalised graphs see: -# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key - -[Definition] - -actionban = curl --fail --user-agent "" http://www.badips.com/add// - -[Init] - -# Option: category -# Notes.: Values are from the list here: http://www.badips.com/get/categories -category = diff --git a/config/action.d/badips.py b/config/action.d/badips.py deleted file mode 100644 index 805120e9..00000000 --- a/config/action.d/badips.py +++ /dev/null @@ -1,391 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import sys -if sys.version_info < (2, 7): # pragma: no cover - raise ImportError("badips.py action requires Python >= 2.7") -import json -import threading -import logging -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.request import Request, urlopen - from urllib.parse import urlencode - from urllib.error import HTTPError -else: # pragma: 3.x no cover - from urllib2 import Request, urlopen, HTTPError - from urllib import urlencode - -from fail2ban.server.actions import Actions, ActionBase, BanTicket -from fail2ban.helpers import splitwords, str2LogLevel - - - -class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable - """Fail2Ban action which reports bans to badips.com, and also - blacklist bad IPs listed on badips.com by using another action's - ban method. - - Parameters - ---------- - jail : Jail - The jail which the action belongs to. - name : str - Name assigned to the action. - category : str - Valid badips.com category for reporting failures. - score : int, optional - Minimum score for bad IPs. Default 3. - age : str, optional - Age of last report for bad IPs, per badips.com syntax. - Default "24h" (24 hours) - banaction : str, optional - Name of banaction to use for blacklisting bad IPs. If `None`, - no blacklist of IPs will take place. - Default `None`. - bancategory : str, optional - Name of category to use for blacklisting, which can differ - from category used for reporting. e.g. may want to report - "postfix", but want to use whole "mail" category for blacklist. - Default `category`. - bankey : str, optional - Key issued by badips.com to retrieve personal list - of blacklist IPs. - updateperiod : int, optional - Time in seconds between updating bad IPs blacklist. - Default 900 (15 minutes) - loglevel : int/str, optional - Log level of the message when an IP is (un)banned. - Default `DEBUG`. - Can be also supplied as two-value list (comma- or space separated) to - provide level of the summary message when a group of IPs is (un)banned. - Example `DEBUG,INFO`. - agent : str, optional - User agent transmitted to server. - Default `Fail2Ban/ver.` - - Raises - ------ - ValueError - If invalid `category`, `score`, `banaction` or `updateperiod`. - """ - - TIMEOUT = 10 - _badips = "https://www.badips.com" - def _Request(self, url, **argv): - return Request(url, headers={'User-Agent': self.agent}, **argv) - - def __init__(self, jail, name, category, score=3, age="24h", - banaction=None, bancategory=None, bankey=None, updateperiod=900, - loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): - super(BadIPsAction, self).__init__(jail, name) - - self.timeout = timeout - self.agent = agent - self.category = category - self.score = score - self.age = age - self.banaction = banaction - self.bancategory = bancategory or category - self.bankey = bankey - loglevel = splitwords(loglevel) - self.sumloglevel = str2LogLevel(loglevel[-1]) - self.loglevel = str2LogLevel(loglevel[0]) - self.updateperiod = updateperiod - - self._bannedips = set() - # Used later for threading.Timer for updating badips - self._timer = None - - @staticmethod - def isAvailable(timeout=1): - try: - response = urlopen(Request("/".join([BadIPsAction._badips]), - headers={'User-Agent': "Fail2Ban"}), timeout=timeout) - return True, '' - except Exception as e: # pragma: no cover - return False, e - - def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) - messages = {} - try: - messages = json.loads(response.read().decode('utf-8')) - except: - pass - self._logSys.error( - "%s. badips.com response: '%s'", what, - messages.get('err', 'Unknown')) - - def getCategories(self, incParents=False): - """Get badips.com categories. - - Returns - ------- - set - Set of categories. - - Raises - ------ - HTTPError - Any issues with badips.com request. - ValueError - If badips.com response didn't contain necessary information - """ - try: - response = urlopen( - self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch categories") - raise - else: - response_json = json.loads(response.read().decode('utf-8')) - if not 'categories' in response_json: - err = "badips.com response lacked categories specification. Response was: %s" \ - % (response_json,) - self._logSys.error(err) - raise ValueError(err) - categories = response_json['categories'] - categories_names = set( - value['Name'] for value in categories) - if incParents: - categories_names.update(set( - value['Parent'] for value in categories - if "Parent" in value)) - return categories_names - - def getList(self, category, score, age, key=None): - """Get badips.com list of bad IPs. - - Parameters - ---------- - category : str - Valid badips.com category. - score : int - Minimum score for bad IPs. - age : str - Age of last report for bad IPs, per badips.com syntax. - key : str, optional - Key issued by badips.com to fetch IPs reported with the - associated key. - - Returns - ------- - set - Set of bad IPs. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "?".join([ - "/".join([self._badips, "get", "list", category, str(score)]), - urlencode({'age': age})]) - if key: - url = "&".join([url, urlencode({'key': key})]) - self._logSys.debug('badips.com: get list, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch bad IP list") - raise - else: - return set(response.read().decode('utf-8').split()) - - @property - def category(self): - """badips.com category for reporting IPs. - """ - return self._category - - @category.setter - def category(self, category): - if category not in self.getCategories(): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - category) - raise ValueError("Invalid category: %s" % category) - self._category = category - - @property - def bancategory(self): - """badips.com bancategory for fetching IPs. - """ - return self._bancategory - - @bancategory.setter - def bancategory(self, bancategory): - if bancategory != "any" and bancategory not in self.getCategories(incParents=True): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - bancategory) - raise ValueError("Invalid bancategory: %s" % bancategory) - self._bancategory = bancategory - - @property - def score(self): - """badips.com minimum score for fetching IPs. - """ - return self._score - - @score.setter - def score(self, score): - score = int(score) - if 0 <= score <= 5: - self._score = score - else: - raise ValueError("Score must be 0-5") - - @property - def banaction(self): - """Jail action to use for banning/unbanning. - """ - return self._banaction - - @banaction.setter - def banaction(self, banaction): - if banaction is not None and banaction not in self._jail.actions: - self._logSys.error("Action name '%s' not in jail '%s'", - banaction, self._jail.name) - raise ValueError("Invalid banaction") - self._banaction = banaction - - @property - def updateperiod(self): - """Period in seconds between banned bad IPs will be updated. - """ - return self._updateperiod - - @updateperiod.setter - def updateperiod(self, updateperiod): - updateperiod = int(updateperiod) - if updateperiod > 0: - self._updateperiod = updateperiod - else: - raise ValueError("Update period must be integer greater than 0") - - def _banIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].ban(ai) - except Exception as e: - self._logSys.error( - "Error banning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._bannedips.add(ip) - self._logSys.log(self.loglevel, - "Banned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - - def _unbanIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].unban(ai) - except Exception as e: - self._logSys.error( - "Error unbanning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._logSys.log(self.loglevel, - "Unbanned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - finally: - self._bannedips.remove(ip) - - def start(self): - """If `banaction` set, blacklists bad IPs. - """ - if self.banaction is not None: - self.update() - - def update(self): - """If `banaction` set, updates blacklisted IPs. - - Queries badips.com for list of bad IPs, removing IPs from the - blacklist if no longer present, and adds new bad IPs to the - blacklist. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - - try: - ips = self.getList( - self.bancategory, self.score, self.age, self.bankey) - # Remove old IPs no longer listed - s = self._bannedips - ips - m = len(s) - self._unbanIPs(s) - # Add new IPs which are now listed - s = ips - self._bannedips - p = len(s) - self._banIPs(s) - if m != 0 or p != 0: - self._logSys.log(self.sumloglevel, - "Updated IPs for jail '%s' (-%d/+%d)", - self._jail.name, m, p) - self._logSys.debug( - "Next update for jail '%' in %i seconds", - self._jail.name, self.updateperiod) - finally: - self._timer = threading.Timer(self.updateperiod, self.update) - self._timer.start() - - def stop(self): - """If `banaction` set, clears blacklisted IPs. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - self._unbanIPs(self._bannedips.copy()) - - def ban(self, aInfo): - """Reports banned IP to badips.com. - - Parameters - ---------- - aInfo : dict - Dictionary which includes information in relation to - the ban. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - self._logSys.debug('badips.com: ban, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to ban") - raise - else: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.debug( - "Response from badips.com report: '%s'", - messages['suc']) - -Action = BadIPsAction diff --git a/config/jail.conf b/config/jail.conf index ddbcf61e..be035112 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -204,20 +204,6 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. -# -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] - # Report ban via abuseipdb.com. # # See action.d/abuseipdb.conf for usage example and details. diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py deleted file mode 100644 index 013c0fdb..00000000 --- a/fail2ban/tests/action_d/test_badips.py +++ /dev/null @@ -1,157 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import unittest -import sys -from functools import wraps -from socket import timeout -from ssl import SSLError - -from ..actiontestcase import CallingMap -from ..dummyjail import DummyJail -from ..servertestcase import IPAddr -from ..utils import LogCaptureTestCase, CONFIG_DIR - -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.error import HTTPError, URLError -else: # pragma: 3.x no cover - from urllib2 import HTTPError, URLError - -def skip_if_not_available(f): - """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". - """ - @wraps(f) - def wrapper(self, *args): - try: - return f(self, *args) - except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues - if not isinstance(e, timeout) and 'timed out' not in str(e): - if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: - raise - raise unittest.SkipTest('Skip test because of %s' % e) - return wrapper - -if sys.version_info >= (2,7): # pragma: no cover - may be unavailable - class BadIPsActionTest(LogCaptureTestCase): - - available = True, None - pythonModule = None - modAction = None - - @skip_if_not_available - def setUp(self): - """Call before every test case.""" - super(BadIPsActionTest, self).setUp() - unittest.F2B.SkipIfNoNetwork() - - self.jail = DummyJail() - - self.jail.actions.add("test") - - pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py") - - # check availability (once if not alive, used shorter timeout as in test cases): - if BadIPsActionTest.available[0]: - if not BadIPsActionTest.modAction: - if not BadIPsActionTest.pythonModule: - BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action - self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) - if not BadIPsActionTest.available[0]: - raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) - - self.jail.actions.add("badips", pythonModuleName, initOpts={ - 'category': "ssh", - 'banaction': "test", - 'age': "2w", - 'score': 5, - #'key': "fail2ban-test-suite", - #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 60), - }) - self.action = self.jail.actions["badips"] - - def tearDown(self): - """Call after every test case.""" - # Must cancel timer! - if self.action._timer: - self.action._timer.cancel() - super(BadIPsActionTest, self).tearDown() - - @skip_if_not_available - def testCategory(self): - categories = self.action.getCategories() - self.assertIn("ssh", categories) - self.assertTrue(len(categories) >= 10) - - self.assertRaises( - ValueError, setattr, self.action, "category", - "invalid-category") - - # Not valid for reporting category... - self.assertRaises( - ValueError, setattr, self.action, "category", "mail") - # but valid for blacklisting. - self.action.bancategory = "mail" - - @skip_if_not_available - def testScore(self): - self.assertRaises(ValueError, setattr, self.action, "score", -5) - self.action.score = 3 - self.action.score = "3" - - @skip_if_not_available - def testBanaction(self): - self.assertRaises( - ValueError, setattr, self.action, "banaction", - "invalid-action") - self.action.banaction = "test" - - @skip_if_not_available - def testUpdateperiod(self): - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", -50) - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", 0) - self.action.updateperiod = 900 - self.action.updateperiod = "900" - - @skip_if_not_available - def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - - @skip_if_not_available - def testBanIP(self): - aInfo = CallingMap({ - 'ip': IPAddr('192.0.2.1') - }) - self.action.ban(aInfo) - self.assertLogged('badips.com: ban', wait=True) - self.pruneLog() - # produce an error using wrong category/IP: - self.action._category = 'f2b-this-category-dont-available-test-suite-only' - aInfo['ip'] = '' - self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo) - self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e92edd48..4029c753 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -458,8 +458,6 @@ class JailReaderTest(LogCaptureTestCase): ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), ('action', '%(action_blocklist_de)s\n' - '%(action_badips_report)s\n' - '%(action_badips)s\n' 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' ), )) @@ -473,16 +471,14 @@ class JailReaderTest(LogCaptureTestCase): if len(cmd) <= 4: continue # differentiate between set and multi-set (wrop it here to single set): - if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')): + if cmd[0] == 'set' and cmd[4] == 'agent': act.append(cmd) elif cmd[0] == 'multi-set': act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent']) useragent = 'Fail2Ban/%s' % version - self.assertEqual(len(act), 4) + self.assertEqual(len(act), 2) self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) - self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) - self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) - self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) @with_tmpdir def testGlob(self, d): From 63acc862b139bb1d8b45edb0b3716044b36c7113 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2021 18:21:42 +0100 Subject: [PATCH 039/139] `action.d/nginx-block-map.conf`: reload nginx only if it is running (also avoid error in nginx-errorlog, gh-2949) and better test coverage for the action --- config/action.d/nginx-block-map.conf | 11 +++++++++-- fail2ban/tests/fail2banclienttestcase.py | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index ee702907..0de382bd 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -84,8 +84,15 @@ srv_cfg_path = /etc/nginx/ #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf srv_cmd = nginx -# first test configuration is correct, hereafter send reload signal: -blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then +# pid file (used to check nginx is running): +srv_pid = /run/nginx.pid + +# command used to check whether nginx is running and configuration is valid: +srv_is_running = [ -f "%(srv_pid)s" ] +srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt + +# first test nginx is running and configuration is correct, hereafter send reload signal: +blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi; fi; diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 03b1d7ce..d1aec5ab 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1281,7 +1281,7 @@ class Fail2banServerTest(Fail2banClientServerBase): 'backend = polling', 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', - 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + 'action = nginx-block-map[srv_cmd="echo nginx", srv_pid="%(tmp)s/f2b.pid", blck_lst_file="%(tmp)s/blck-lst.map"]', ' blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; \', email="Fail2Ban ", ' 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=]', 'filter =', @@ -1321,6 +1321,8 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check nginx reload is logged (pid of fail2ban is used to simulate success check nginx is running): + self.assertLogged("stdout: 'nginx -qt'", "stdout: 'nginx -s reload'", all=True) # check blocklist_de substitution (e. g. new-line after ): self.assertLogged( "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " From a45b1c974c969e9486326872447c33feca9ce0b2 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 2 Mar 2021 19:24:03 +0100 Subject: [PATCH 040/139] filter.d/ignorecommands/apache-fakegooglebot: added timeout parameter (default 55 seconds) - avoid fail with timeout (default 1 minute) by reverse lookup on some slow DNS services (googlebots must be resolved fast); closes gh-2951 --- .../ignorecommands/apache-fakegooglebot | 25 +++++++++++++------ fail2ban/tests/filtertestcase.py | 13 +++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index b11f0d98..8351efa2 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -6,24 +6,35 @@ # import sys from fail2ban.server.ipdns import DNSUtils, IPAddr +from threading import Thread def process_args(argv): - if len(argv) != 2: - raise ValueError("Please provide a single IP as an argument. Got: %s\n" - % (argv[1:])) + if len(argv) - 1 not in (1, 2): + raise ValueError("Usage %s ip ?timeout?. Got: %s\n" + % (argv[0], argv[1:])) ip = argv[1] if not IPAddr(ip).isValid: raise ValueError("Argument must be a single valid IP. Got: %s\n" % ip) - return ip + return argv[1:] google_ips = None -def is_googlebot(ip): +def is_googlebot(ip, timeout=55): import re - host = DNSUtils.ipToName(ip) + timeout = float(timeout or 0) + if timeout: + def ipToNameTO(host, ip, timeout): + host[0] = DNSUtils.ipToName(ip) + host = [None] + th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start() + th.join(timeout) + host = host[0] + else: + host = DNSUtils.ipToName(ip) + if not host or not re.match(r'.*\.google(bot)?\.com$', host): return False host_ips = DNSUtils.dnsToIp(host) @@ -31,7 +42,7 @@ def is_googlebot(ip): if __name__ == '__main__': # pragma: no cover try: - ret = is_googlebot(process_args(sys.argv)) + ret = is_googlebot(*process_args(sys.argv)) except ValueError as e: sys.stderr.write(str(e)) sys.exit(2) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b9b7e8aa..4f716663 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -606,13 +606,14 @@ class IgnoreIPDNS(LogCaptureTestCase): cmd = os.path.join(STOCK_CONF_DIR, "filter.d/ignorecommands/apache-fakegooglebot") ## below test direct as python module: mod = Utils.load_python_module(cmd) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "128.178.222.69"]))) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "128.178.222.69"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1", 0.1]))) bot_ips = ['66.249.66.1'] for ip in bot_ips: - self.assertTrue(mod.is_googlebot(mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd]))) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"]))) + self.assertTrue(mod.is_googlebot(*mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd]))) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd, "192.0"]))) ## via command: self.filter.ignoreCommand = cmd + " " for ip in bot_ips: @@ -624,7 +625,7 @@ class IgnoreIPDNS(LogCaptureTestCase): self.pruneLog() self.filter.ignoreCommand = cmd + " bad arguments " self.assertFalse(self.filter.inIgnoreIPList("192.0")) - self.assertLogged('Please provide a single IP as an argument.') + self.assertLogged('Usage') From 04aba6168c5f9b4b7b2bc4e2f4d128b62c3b7633 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Mar 2021 13:02:00 +0100 Subject: [PATCH 041/139] fixed typo, `--` is not expected in options declaration, so `--dump-pretty` did never work (only `--dp` is working) --- fail2ban/client/fail2bancmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 03683cad..c2f6d0be 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -192,7 +192,7 @@ class Fail2banCmdLine(): cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'conf=', 'pidfile=', 'pname=', 'socket=', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() From df5e024fb8819e90ee2b3435eb4f519e77c26495 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Mar 2021 20:13:04 +0100 Subject: [PATCH 042/139] new issue templates --- .github/ISSUE_TEMPLATE.md | 49 ---------------- .github/ISSUE_TEMPLATE/bug_report.md | 70 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 35 ++++++++++++ .github/ISSUE_TEMPLATE/filter_request.md | 59 +++++++++++++++++++ 4 files changed, 164 insertions(+), 49 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/filter_request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index cb4b4bc6..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,49 +0,0 @@ -_We will be very grateful, if your problem was described as completely as possible, -enclosing excerpts from logs (if possible within DEBUG mode, if no errors evident -within INFO mode), and configuration in particular of effected relevant settings -(e.g., with ` fail2ban-client -d | grep 'affected-jail-name' ` for a particular -jail troubleshooting). -Thank you in advance for the details, because such issues like "It does not work" -alone could not help to resolve anything! -Thanks! (remove this paragraph and other comments upon reading)_ - -### Environment: - -_Fill out and check (`[x]`) the boxes which apply. If your Fail2Ban version is outdated, -and you can't verify that the issue persists in the recent release, better seek support -from the distribution you obtained Fail2Ban from_ - -- Fail2Ban version (including any possible distribution suffixes): -- OS, including release name/version: -- [ ] Fail2Ban installed via OS/distribution mechanisms -- [ ] You have not applied any additional foreign patches to the codebase -- [ ] Some customizations were done to the configuration (provide details below is so) - -### The issue: - -_Summary here_ - -#### Steps to reproduce - -#### Expected behavior - -#### Observed behavior - -#### Any additional information - -### Configuration, dump and another helpful excerpts - -#### Any customizations done to /etc/fail2ban/ configuration -``` -``` - -#### Relevant parts of /var/log/fail2ban.log file: -_preferably obtained while running fail2ban with `loglevel = 4`_ - -``` -``` - -#### Relevant lines from monitored log files in question: - -``` -``` \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..33d94e10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,70 @@ +--- +name: Bug report +about: Report a bug within the fail2ban engines (not filters or jails) +title: '[BR]: ' +labels: bug +assignees: '' + +--- + + + +### Environment: + + + +- Fail2Ban version : +- OS, including release name/version : +- [ ] Fail2Ban installed via OS/distribution mechanisms +- [ ] You have not applied any additional foreign patches to the codebase +- [ ] Some customizations were done to the configuration (provide details below is so) + +### The issue: + + + +#### Steps to reproduce + +#### Expected behavior + +#### Observed behavior + +#### Any additional information + + +### Configuration, dump and another helpful excerpts + +#### Any customizations done to /etc/fail2ban/ configuration + +``` +``` + +#### Relevant parts of /var/log/fail2ban.log file: + + +``` +``` + +#### Relevant lines from monitored log files: + +``` +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..41812e82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,35 @@ +--- +name: Feature request +about: Suggest an idea or an enhancement for this project +title: '[RFE]: ' +labels: enhancement +assignees: '' + +--- + + + +#### Feature request type + + +#### Description + + +#### Considered alternatives + + +#### Any additional information + diff --git a/.github/ISSUE_TEMPLATE/filter_request.md b/.github/ISSUE_TEMPLATE/filter_request.md new file mode 100644 index 00000000..eeaba5ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/filter_request.md @@ -0,0 +1,59 @@ +--- +name: Filter Request +about: Request a new jail or filter to be supported or existing filter extended with new failregex. +title: '[FR]: ' +labels: filter-request +assignees: '' + +--- + + + +### Environment: + + + +- Fail2Ban version : +- OS, including release name/version : + +#### Service, project or product which log or journal should be monitored + +- Name of filter or jail in Fail2Ban (if already exists) : +- Service, project or product name, including release name/version : +- Repository or URL (if known) : +- Service type : +- Ports and protocols the service is listening : + +#### Log or journal information + + + + +- Log file name(s) : + + + +- Journal identifier or unit name : + +#### Any additional information + + +### Relevant lines from monitored log files: + +#### failures in sense of fail2ban filter (fail2ban must match): + +``` +``` + +#### legitimate messages (fail2ban should not consider as failures): + +``` +``` From 08393f9d82d4929ce3ee09bd9f3c685d38b2428d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 3 Mar 2021 20:28:27 +0100 Subject: [PATCH 043/139] Update filter_request.md --- .github/ISSUE_TEMPLATE/filter_request.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/filter_request.md b/.github/ISSUE_TEMPLATE/filter_request.md index eeaba5ae..caf02f90 100644 --- a/.github/ISSUE_TEMPLATE/filter_request.md +++ b/.github/ISSUE_TEMPLATE/filter_request.md @@ -1,6 +1,6 @@ --- -name: Filter Request -about: Request a new jail or filter to be supported or existing filter extended with new failregex. +name: Filter request +about: Request a new jail or filter to be supported or existing filter extended with new failregex title: '[FR]: ' labels: filter-request assignees: '' From f15ed356198728c18470794ce6d88fb786571dc4 Mon Sep 17 00:00:00 2001 From: Mike Gabriel Date: Thu, 25 Feb 2021 20:13:18 +0100 Subject: [PATCH 044/139] config/: Add support for filtering out detected port scans via scanlogd. --- config/filter.d/scanlogd.conf | 17 +++++++++++++++++ config/jail.conf | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 config/filter.d/scanlogd.conf diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf new file mode 100644 index 00000000..65ad63f6 --- /dev/null +++ b/config/filter.d/scanlogd.conf @@ -0,0 +1,17 @@ +# Fail2Ban filter for port scans detected by scanlogd + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] + +_daemon = scanlogd + +failregex = ^%(__prefix_line)s\ to\ [\.:0-9a-f]+\ ports\ [\ \.,0-9]+,\ f.......,\ TOS\ [0-9]+,\ TTL\ [0-9]+\ \@[0-9]{1,2}:[0-9]{2}:[0-9]{2}$ + +ignoreregex = + +# Author: Mike Gabriel diff --git a/config/jail.conf b/config/jail.conf index 28f259a0..d6d8af67 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -965,3 +965,6 @@ logpath = %(apache_error_log)s # see `filter.d/traefik-auth.conf` for details and service example. port = http,https logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %{syslog_local0} From 529866b2bb41586342652af19aaca7435ab28c28 Mon Sep 17 00:00:00 2001 From: oukb <5672797+oukb@users.noreply.github.com> Date: Mon, 8 Mar 2021 19:14:28 +0300 Subject: [PATCH 045/139] nsd.conf: fix for the current log format New nsd 4.3.5 log format: | [2021-03-05 05:25:14.562] nsd[160800]: info: axfr for example.com. from 192.35.168.32 refused, no acl matches | [2021-03-06 05:24:33.223] nsd[356033]: info: axfr for localhost. from 192.35.168.160 refused, no acl matches | [2021-03-07 05:23:26.641] nsd[547893]: info: axfr for example.com. from 192.35.168.64 refused, no acl matches | [2021-03-08 05:18:54.067] nsd[739606]: info: axfr for example.com. from 192.35.168.32 refused, no acl matches --- config/filter.d/nsd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index bfd99544..9399db23 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -23,9 +23,9 @@ _daemon = nsd # Values: TEXT failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ - ^%(__prefix_line)sinfo: .* refused, no acl matches\.$ + ^%(__prefix_line)sinfo: .* refused, no acl matches ignoreregex = datepattern = {^LN-BEG}Epoch - {^LN-BEG} \ No newline at end of file + {^LN-BEG} From 725354c79315d8eac6cdc7ffa550a8acb9a01d73 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 20 Mar 2021 22:33:31 +0100 Subject: [PATCH 046/139] action info extended with new members for jail info (usable as tags in command actions): `jail.found`, `jail.found_total` - current and total found failures `jail.banned`, `jail.banned_total` - current and total bans closes #10 --- fail2ban/server/actions.py | 55 ++++++++++--------- fail2ban/tests/databasetestcase.py | 8 ++- fail2ban/tests/fail2banclienttestcase.py | 13 +++-- .../tests/files/action.d/action_checkainfo.py | 3 + 4 files changed, 48 insertions(+), 31 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 91e1ebaf..e07ffb17 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -84,7 +84,7 @@ class Actions(JailThread, Mapping): self._jail = jail self._actions = OrderedDict() ## The ban manager. - self.__banManager = BanManager() + self.banManager = BanManager() self.banEpoch = 0 self.__lastConsistencyCheckTM = 0 ## Precedence of ban (over unban), so max number of tickets banned (to call an unban check): @@ -203,7 +203,7 @@ class Actions(JailThread, Mapping): def setBanTime(self, value): value = MyTime.str2seconds(value) - self.__banManager.setBanTime(value) + self.banManager.setBanTime(value) logSys.info(" banTime: %s" % value) ## @@ -212,10 +212,10 @@ class Actions(JailThread, Mapping): # @return the time def getBanTime(self): - return self.__banManager.getBanTime() + return self.banManager.getBanTime() def getBanned(self, ids): - lst = self.__banManager.getBanList() + lst = self.banManager.getBanList() if not ids: return lst if len(ids) == 1: @@ -230,7 +230,7 @@ class Actions(JailThread, Mapping): list The list of banned IP addresses. """ - return self.__banManager.getBanList(ordered=True, withTime=withTime) + return self.banManager.getBanList(ordered=True, withTime=withTime) def addBannedIP(self, ip): """Ban an IP or list of IPs.""" @@ -282,7 +282,7 @@ class Actions(JailThread, Mapping): if db and self._jail.database is not None: self._jail.database.delBan(self._jail, ip) # Find the ticket with the IP. - ticket = self.__banManager.getTicketByID(ip) + ticket = self.banManager.getTicketByID(ip) if ticket is not None: # Unban the IP. self.__unBan(ticket) @@ -291,7 +291,7 @@ class Actions(JailThread, Mapping): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter(ipa.contains, self.__banManager.getBanList()) + ips = filter(ipa.contains, self.banManager.getBanList()) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: @@ -350,7 +350,7 @@ class Actions(JailThread, Mapping): continue # wait for ban (stop if gets inactive, pending ban or unban): bancnt = 0 - wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + wt = min(self.sleeptime, self.banManager._nextUnbanTime - MyTime.time()) logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): bancnt = self.__checkBan() @@ -397,7 +397,12 @@ class Actions(JailThread, Mapping): "ipfailures": lambda self: self._mi4ip(True).getAttempt(), "ipjailfailures": lambda self: self._mi4ip().getAttempt(), # raw ticket info: - "raw-ticket": lambda self: repr(self.__ticket) + "raw-ticket": lambda self: repr(self.__ticket), + # jail info: + "jail.banned": lambda self: self.__jail.actions.banManager.size(), + "jail.banned_total": lambda self: self.__jail.actions.banManager.getBanTotal(), + "jail.found": lambda self: self.__jail.filter.failManager.size(), + "jail.found_total": lambda self: self.__jail.filter.failManager.getFailTotal() } __slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip') @@ -494,11 +499,11 @@ class Actions(JailThread, Mapping): for ticket in tickets: bTicket = BanTicket.wrap(ticket) - btime = ticket.getBanTime(self.__banManager.getBanTime()) + btime = ticket.getBanTime(self.banManager.getBanTime()) ip = bTicket.getIP() aInfo = self._getActionInfo(bTicket) reason = {} - if self.__banManager.addBanTicket(bTicket, reason=reason): + if self.banManager.addBanTicket(bTicket, reason=reason): cnt += 1 # report ticket to observer, to check time should be increased and hereafter observer writes ban to database (asynchronous) if Observers.Main is not None and not bTicket.restored: @@ -557,7 +562,7 @@ class Actions(JailThread, Mapping): # and increase ticket time if "bantime.increment" set) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, - self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) + self.banManager.getBanTotal(), self.banManager.size(), self._jail.name) return cnt def __reBan(self, ticket, actions=None, log=True): @@ -597,7 +602,7 @@ class Actions(JailThread, Mapping): def _prolongBan(self, ticket): # prevent to prolong ticket that was removed in-between, # if it in ban list - ban time already prolonged (and it stays there): - if not self.__banManager._inBanList(ticket): return + if not self.banManager._inBanList(ticket): return # do actions : aInfo = None for name, action in self._actions.iteritems(): @@ -622,13 +627,13 @@ class Actions(JailThread, Mapping): Unban IP addresses which are outdated. """ - lst = self.__banManager.unBanList(MyTime.time(), maxCount) + lst = self.banManager.unBanList(MyTime.time(), maxCount) for ticket in lst: self.__unBan(ticket) cnt = len(lst) if cnt: logSys.debug("Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt def __flushBan(self, db=False, actions=None, stop=False): @@ -642,10 +647,10 @@ class Actions(JailThread, Mapping): log = True if actions is None: logSys.debug(" Flush ban list") - lst = self.__banManager.flushBanList() + lst = self.banManager.flushBanList() else: log = False # don't log "[jail] Unban ..." if removing actions only. - lst = iter(self.__banManager) + lst = iter(self.banManager) cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} @@ -682,7 +687,7 @@ class Actions(JailThread, Mapping): self.__unBan(ticket, actions=actions, log=log) cnt += 1 logSys.debug(" Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt def __unBan(self, ticket, actions=None, log=True): @@ -725,18 +730,18 @@ class Actions(JailThread, Mapping): logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) if flavor != "short": - banned = self.__banManager.getBanList() + banned = self.banManager.getBanList() cnt = len(banned) else: - cnt = self.__banManager.size() + cnt = self.banManager.size() ret = [("Currently banned", cnt), - ("Total banned", self.__banManager.getBanTotal())] + ("Total banned", self.banManager.getBanTotal())] if flavor != "short": ret += [("Banned IP list", banned)] if flavor == "cymru": - cymru_info = self.__banManager.getBanListExtendedCymruInfo() + cymru_info = self.banManager.getBanListExtendedCymruInfo() ret += \ - [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), - ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), - ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + [("Banned ASN list", self.banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.banManager.geBanListExtendedRIR(cymru_info))] return ret diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index a8e2ceae..298730ae 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -29,7 +29,7 @@ import tempfile import sqlite3 import shutil -from ..server.filter import FileContainer +from ..server.filter import FileContainer, Filter from ..server.mytime import MyTime from ..server.ticket import FailTicket from ..server.actions import Actions, Utils @@ -544,17 +544,21 @@ class DatabaseTest(LogCaptureTestCase): self.testAddJail() # Jail required self.jail.database = self.db self.db.addJail(self.jail) - actions = Actions(self.jail) + actions = self.jail.actions actions.add( "action_checkainfo", os.path.join(TEST_FILES_DIR, "action.d/action_checkainfo.py"), {}) + actions.banManager.setBanTotal(20) + self.jail._Jail__filter = flt = Filter(self.jail) + flt.failManager.setFailTotal(50) ticket = FailTicket("1.2.3.4") ticket.setAttempt(5) ticket.setMatches(['test', 'test']) self.jail.putFailTicket(ticket) actions._Actions__checkBan() self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) + self.assertLogged("jail info %d, %d, %d, %d" % (1, 21, 0, 50)) def testDelAndAddJail(self): self.testAddJail() # Add jail diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 244d23b0..5d322f70 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1410,8 +1410,9 @@ class Fail2banServerTest(Fail2banClientServerBase): 'jails': ( # default: '''test_action = dummy[actionstart_on_demand=1, init="start: %(__name__)s", target="%(tmp)s/test.txt", - actionban='; - echo ""; printf "=====\\n%%b\\n=====\\n\\n" "" >> ']''', + actionban='; echo "found: / , banned: / " + echo ""; printf "=====\\n%%b\\n=====\\n\\n" "" >> ', + actionstop='; echo "stats - found: , banned: "']''', # jail sendmail-auth: '[sendmail-auth]', 'backend = polling', @@ -1456,7 +1457,8 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_file(lgfn, "w+", *smaut_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( - "[sendmail-auth] Ban 192.0.2.1", "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) + "[sendmail-auth] Ban 192.0.2.1", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) _out_file(tofn) td = _read_file(tofn) # check matches (maxmatches = 2, so only 2 & 3 available): @@ -1470,7 +1472,8 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_file(lgfn, "w+", *smrej_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( - "[sendmail-reject] Ban 192.0.2.2", "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) + "[sendmail-reject] Ban 192.0.2.2", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) _out_file(tofn) td = _read_file(tofn) # check matches (no maxmatches, so all matched messages are available): @@ -1484,6 +1487,8 @@ class Fail2banServerTest(Fail2banClientServerBase): # wait a bit: self.assertLogged( "Reload finished.", + "stdout: 'stats sendmail-auth - found: 3, banned: 1'", + "stdout: 'stats sendmail-reject - found: 3, banned: 1'", "[sendmail-auth] Restore Ban 192.0.2.1", "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) # check matches again - (dbmaxmatches = 1), so it should be only last match after restart: td = _read_file(tofn) diff --git a/fail2ban/tests/files/action.d/action_checkainfo.py b/fail2ban/tests/files/action.d/action_checkainfo.py index 63dd4f5b..c5eaf0f8 100644 --- a/fail2ban/tests/files/action.d/action_checkainfo.py +++ b/fail2ban/tests/files/action.d/action_checkainfo.py @@ -8,6 +8,9 @@ class TestAction(ActionBase): self._logSys.info("ban ainfo %s, %s, %s, %s", aInfo["ipmatches"] != '', aInfo["ipjailmatches"] != '', aInfo["ipfailures"] > 0, aInfo["ipjailfailures"] > 0 ) + self._logSys.info("jail info %d, %d, %d, %d", + aInfo["jail.banned"], aInfo["jail.banned_total"], aInfo["jail.found"], aInfo["jail.found_total"] + ) def unban(self, aInfo): pass From 9bdc4be6cce7398cf51d811b7d71e1999c262fb3 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 21 Mar 2021 23:35:09 +0100 Subject: [PATCH 047/139] stability: better recognition of rotation (e. g. on hash collision, consider current size and last known position now), no hash of empty file (or not fulfilled line), etc; performance: avoid unnecessary seek to start of file and hash calculation - now it occurs only if file really rotated (ino changing or size shrinking), otherwise not earlier than in 30 seconds; avoid unneeded log-rotation in tests --- fail2ban/server/database.py | 2 +- fail2ban/server/filter.py | 109 ++++++++++++++--------- fail2ban/tests/databasetestcase.py | 7 +- fail2ban/tests/fail2banclienttestcase.py | 16 ++-- fail2ban/tests/filtertestcase.py | 19 ++-- 5 files changed, 88 insertions(+), 65 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index ed736a7a..86b0ea68 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -502,7 +502,7 @@ class Fail2BanDb(object): except TypeError: firstLineMD5 = None - if not firstLineMD5 and (pos or md5): + if firstLineMD5 is None and (pos or md5 is not None): cur.execute( "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 7ad8a462..79cde6b5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1155,6 +1155,8 @@ class FileFilter(Filter): if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Seek to find time %s (%s), file size %s", date, MyTime.time2str(date), fs) + if not fs: + return minp = container.getPos() maxp = fs tryPos = minp @@ -1281,20 +1283,25 @@ class FileContainer: self.setEncoding(encoding) self.__tail = tail self.__handler = None + self.__pos = 0 + self.__pos4hash = 0 + self.__hash = '' + self.__hashNextTime = time.time() + 30 # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') - stats = os.fstat(handler.fileno()) - self.__ino = stats.st_ino try: - firstLine = handler.readline() - # Computes the MD5 of the first line. - self.__hash = md5sum(firstLine).hexdigest() - # Start at the beginning of file if tail mode is off. - if tail: - handler.seek(0, 2) - self.__pos = handler.tell() - else: - self.__pos = 0 + stats = os.fstat(handler.fileno()) + self.__ino = stats.st_ino + if stats.st_size: + firstLine = handler.readline() + # first line available and contains new-line: + if firstLine != firstLine.rstrip('\r\n'): + # Computes the MD5 of the first line. + self.__hash = md5sum(firstLine).hexdigest() + # if tail mode scroll to the end of file + if tail: + handler.seek(0, 2) + self.__pos = handler.tell() finally: handler.close() ## shows that log is in operation mode (expecting new messages only from here): @@ -1304,6 +1311,10 @@ class FileContainer: return self.__filename def getFileSize(self): + h = self.__handler + if h is not None: + stats = os.fstat(h.fileno()) + return stats.st_size return os.path.getsize(self.__filename); def setEncoding(self, encoding): @@ -1322,38 +1333,54 @@ class FileContainer: def setPos(self, value): self.__pos = value - def open(self): - self.__handler = open(self.__filename, 'rb') - # Set the file descriptor to be FD_CLOEXEC - fd = self.__handler.fileno() - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - # Stat the file before even attempting to read it - stats = os.fstat(self.__handler.fileno()) - if not stats.st_size: - # yoh: so it is still an empty file -- nothing should be - # read from it yet - # print "D: no content -- return" - return False - firstLine = self.__handler.readline() - # Computes the MD5 of the first line. - myHash = md5sum(firstLine).hexdigest() - ## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % ( - ## self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos, - ## self.__hash != myHash or self.__ino != stats.st_ino) - ## sys.stdout.flush() - # Compare hash and inode - if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.log(logging.MSG, "Log rotation detected for %s", self.__filename) - self.__hash = myHash - self.__ino = stats.st_ino - self.__pos = 0 - # Sets the file pointer to the last position. - self.__handler.seek(self.__pos) + def open(self, forcePos=None): + h = open(self.__filename, 'rb') + try: + # Set the file descriptor to be FD_CLOEXEC + fd = h.fileno() + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + myHash = self.__hash + # Stat the file before even attempting to read it + stats = os.fstat(h.fileno()) + rotflg = stats.st_size < self.__pos or stats.st_ino != self.__ino + if rotflg or not len(myHash) or time.time() > self.__hashNextTime: + myHash = '' + firstLine = h.readline() + # Computes the MD5 of the first line (if it is complete) + if firstLine != firstLine.rstrip('\r\n'): + myHash = md5sum(firstLine).hexdigest() + self.__hashNextTime = time.time() + 30 + elif stats.st_size == self.__pos: + myHash = self.__hash + # Compare size, hash and inode + if rotflg or myHash != self.__hash: + if self.__hash != '': + logSys.log(logging.MSG, "Log rotation detected for %s, reason: %r", self.__filename, + (stats.st_size, self.__pos, stats.st_ino, self.__ino, myHash, self.__hash)) + self.__ino = stats.st_ino + self.__pos = 0 + self.__hash = myHash + # if nothing to read from file yet (empty or no new data): + if forcePos is not None: + self.__pos = forcePos + elif stats.st_size <= self.__pos: + return False + # Sets the file pointer to the last position. + h.seek(self.__pos) + # leave file open (to read content): + self.__handler = h; h = None + finally: + # close (no content or error only) + if h: + h.close(); h = None return True def seek(self, offs, endLine=True): h = self.__handler + if h is None: + self.open(offs) + h = self.__handler # seek to given position h.seek(offs, 0) # goto end of next line @@ -1394,14 +1421,12 @@ class FileContainer: self.getFileName(), self.getEncoding(), self.__handler.readline()) def close(self): - if not self.__handler is None: + if self.__handler is not None: # Saves the last position. self.__pos = self.__handler.tell() # Closes the file. self.__handler.close() self.__handler = None - ## print "D: Closed %s with pos %d" % (handler, self.__pos) - ## sys.stdout.flush() _decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 298730ae..6692b238 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -212,19 +212,20 @@ class DatabaseTest(LogCaptureTestCase): self.jail.name in self.db.getJailNames(True), "Jail not added to database") - def testAddLog(self): + def _testAddLog(self): self.testAddJail() # Jail required _, filename = tempfile.mkstemp(".log", "Fail2BanDb_") self.fileContainer = FileContainer(filename, "utf-8") - self.db.addLog(self.jail, self.fileContainer) + pos = self.db.addLog(self.jail, self.fileContainer) + self.assertTrue(pos is None); # unknown previously self.assertIn(filename, self.db.getLogPaths(self.jail)) os.remove(filename) def testUpdateLog(self): - self.testAddLog() # Add log file + self._testAddLog() # Add log file # Write some text filename = self.fileContainer.getFileName() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 5d322f70..0cbda94f 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -230,7 +230,7 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n)) if create_before_start: for n in create_before_start: - _write_file(n % {'tmp': tmp}, 'w', '') + _write_file(n % {'tmp': tmp}, 'w') # parameters (sock/pid and config, increase verbosity, set log, etc.): vvv, llev = (), "INFO" if unittest.F2B.log_level < logging.INFO: # pragma: no cover @@ -937,10 +937,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'broken-jail' skipped, because of wrong configuration", all=True) # enable both jails, 3 logs for jail1, etc... - # truncate test-log - we should not find unban/ban again by reload: self.pruneLog("[test-phase 1b]") _write_jail_cfg(actions=[1,2]) - _write_file(test1log, "w+") if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(test1log) self.execCmd(SUCCESS, startparams, "reload") @@ -1003,7 +1001,7 @@ class Fail2banServerTest(Fail2banClientServerBase): self.pruneLog("[test-phase 2b]") # write new failures: - _write_file(test2log, "w+", *( + _write_file(test2log, "a+", *( (str(int(MyTime.time())) + " error 403 from 192.0.2.2: test 2",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.3: test 2",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 + @@ -1062,10 +1060,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertEqual(self.execCmdDirect(startparams, 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) - # rotate logs: - _write_file(test1log, "w+") - _write_file(test2log, "w+") - # restart jail without unban all: self.pruneLog("[test-phase 2c]") self.execCmd(SUCCESS, startparams, @@ -1183,7 +1177,7 @@ class Fail2banServerTest(Fail2banClientServerBase): # now write failures again and check already banned (jail1 was alive the whole time) and new bans occurred (jail1 was alive the whole time): self.pruneLog("[test-phase 5]") - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 5",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.5: test 5",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.6: test 5",) * 3 @@ -1469,7 +1463,7 @@ class Fail2banServerTest(Fail2banClientServerBase): self.pruneLog("[test-phase sendmail-reject]") # write log: - _write_file(lgfn, "w+", *smrej_msg) + _write_file(lgfn, "a+", *smrej_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( "[sendmail-reject] Ban 192.0.2.2", "stdout: 'found: 0 / 3, banned: 1 / 1'", @@ -1597,7 +1591,7 @@ class Fail2banServerTest(Fail2banClientServerBase): wakeObs = False _observer_wait_before_incrban(lambda: wakeObs) # write again (IP already bad): - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.11: I'm very bad \"hacker\" `` $(echo test)",) * 2 )) # wait for ban: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 28b2d357..799adfd3 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -195,7 +195,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, f, o) -def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line="", lines=None): """Copy lines from one file to another (which might be already open) Returns open fout @@ -212,9 +212,9 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line fin.readline() # Read i = 0 - lines = [] + if not lines: lines = [] while n is None or i < n: - l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break lines.append(l) @@ -222,6 +222,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) + DefLogSys.debug(' ++ write %d test lines', len(lines)) fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases @@ -253,7 +254,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Read/Write i = 0 while n is None or i < n: - l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break journal.send(MESSAGE=l.strip(), **fields) @@ -1136,13 +1137,15 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1, + lines=["Aug 14 11:59:59 [logrotate] rotation 1"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3, + lines=["Aug 14 11:59:59 [logrotate] rotation 2"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1196,7 +1199,7 @@ def get_monitor_failures_testcase(Filter_): os.rename(tmpsub1, tmpsub2 + 'a') os.mkdir(tmpsub1) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 1"]) self.file.close() self._wait4failures(2) @@ -1207,7 +1210,7 @@ def get_monitor_failures_testcase(Filter_): os.mkdir(tmpsub1) self.waitForTicks(2) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 2"]) self.file.close() self._wait4failures(3) From 343ccd7e8a139a4c6526682290dcbf14b4384973 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 21 Mar 2021 23:35:38 +0100 Subject: [PATCH 048/139] small optimization --- fail2ban/server/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 294d147f..8483b013 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -332,11 +332,9 @@ class Utils(): timeout_expr = lambda: time.time() > time0 else: timeout_expr = timeout - if not interval: - interval = Utils.DEFAULT_SLEEP_INTERVAL if timeout_expr(): break - stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME) + stm = min(stm + (interval or Utils.DEFAULT_SLEEP_INTERVAL), Utils.DEFAULT_SLEEP_TIME) time.sleep(stm) return ret From e587526ede759977b8b483deeaa644f45f97cea3 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Mar 2021 00:55:28 +0100 Subject: [PATCH 049/139] tests: add missing constraint (causing incomplete comparison in below cycle if fewer lines as expected was found) --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 4f716663..3cc17fb1 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -188,6 +188,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): utest.assertEqual(len(found), 1) _assert_equal_entries(utest, found[0], output, count) else: + utest.assertEqual(len(found), len(output)) # sort by string representation of ip (multiple failures with different ips): found = sorted(found, key=lambda x: str(x)) output = sorted(output, key=lambda x: str(x)) From 996920cdaa7b678afc64d1bef6a07ed857c81468 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Mar 2021 00:49:55 +0100 Subject: [PATCH 050/139] in operation mode the filter reads only complete lines (ended with new-line) now, otherwise it would wait for end of line (for its completion) --- fail2ban/server/filter.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 79cde6b5..845b069d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1131,14 +1131,14 @@ class FileFilter(Filter): while not self.idle: line = log.readline() if not self.active: break; # jail has been stopped - if not line: + if line is None: # The jail reached the bottom, simply set in operation for this log # (since we are first time at end of file, growing is only possible after modifications): log.inOperation = True break # acquire in operation from log and process: self.inOperation = inOperation if inOperation is not None else log.inOperation - self.processLineAndAdd(line.rstrip('\r\n')) + self.processLineAndAdd(line) finally: log.close() db = self.jail.database @@ -1180,8 +1180,8 @@ class FileFilter(Filter): dateTimeMatch = None nextp = None while True: - line = container.readline() - if not line: + line = container.readline(False) + if line is None: break (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: @@ -1295,7 +1295,7 @@ class FileContainer: if stats.st_size: firstLine = handler.readline() # first line available and contains new-line: - if firstLine != firstLine.rstrip('\r\n'): + if firstLine != firstLine.rstrip(b'\r\n'): # Computes the MD5 of the first line. self.__hash = md5sum(firstLine).hexdigest() # if tail mode scroll to the end of file @@ -1348,7 +1348,7 @@ class FileContainer: myHash = '' firstLine = h.readline() # Computes the MD5 of the first line (if it is complete) - if firstLine != firstLine.rstrip('\r\n'): + if firstLine != firstLine.rstrip(b'\r\n'): myHash = md5sum(firstLine).hexdigest() self.__hashNextTime = time.time() + 30 elif stats.st_size == self.__pos: @@ -1414,15 +1414,35 @@ class FileContainer: line = line.decode(enc, 'replace') return line - def readline(self): + def readline(self, complete=True): + """Read line from file + + In opposite to pythons readline it doesn't return new-line, + so returns either the line if line is complete (and complete=True) or None + if line is not complete (and complete=True) or there is no content to read. + If line is complete (and complete is True), it also shift current known + position to begin of next line. + """ if self.__handler is None: return "" + rl = self.__handler.readline() + if rl == b'': + return None + # trim new-line here and check the line was written complete (contains a new-line): + l = rl.rstrip(b'\r\n') + if self.inOperation and complete: + if l == rl: + # not fulfilled - seek back and return: + self.__handler.seek(self.__pos, 0) + return None + # shift position (to be able to seek back above): + self.__pos += len(rl) return FileContainer.decode_line( - self.getFileName(), self.getEncoding(), self.__handler.readline()) + self.getFileName(), self.getEncoding(), l) def close(self): if self.__handler is not None: - # Saves the last position. + # Saves the last real position. self.__pos = self.__handler.tell() # Closes the file. self.__handler.close() From d135aeea16343152e47191559b17912df9373705 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 14:12:11 +0100 Subject: [PATCH 051/139] fixes restore of original logging withing tests (`LogCaptureTestCase.tearDown`) - python 3 seemed still to log wordy after tear down (setting of log.level does not restore the level for related log objects - e. g. for logger of `fail2ban.jail` etc, so `fail2ban-testcases '(testVersion|testLongName).*servertest'` generating messages in stdout handler in testLongName) --- fail2ban/tests/servertestcase.py | 10 +++++----- fail2ban/tests/utils.py | 8 +++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index b7b9d802..fc505552 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -66,9 +66,12 @@ class TestServer(Server): class TransmitterBase(LogCaptureTestCase): + TEST_SRV_CLASS = TestServer + def setUp(self): """Call before every test case.""" super(TransmitterBase, self).setUp() + self.server = self.TEST_SRV_CLASS() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -157,10 +160,6 @@ class TransmitterBase(LogCaptureTestCase): class Transmitter(TransmitterBase): - def setUp(self): - self.server = TestServer() - super(Transmitter, self).setUp() - def testServerIsNotStarted(self): # so far isStarted only tested but not used otherwise # and here we don't really .start server @@ -893,8 +892,9 @@ class Transmitter(TransmitterBase): class TransmitterLogging(TransmitterBase): + TEST_SRV_CLASS = Server + def setUp(self): - self.server = Server() super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 921427db..e2c4bccc 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -47,7 +47,7 @@ from ..server import asyncserver from ..version import version -logSys = getLogger(__name__) +logSys = getLogger("fail2ban") TEST_NOW = 1124013600 @@ -126,9 +126,6 @@ def getOptParser(doc=""): def initProcess(opts): # Logger: - global logSys - logSys = getLogger("fail2ban") - llev = None if opts.log_level is not None: # pragma: no cover # so we had explicit settings @@ -777,8 +774,9 @@ class LogCaptureTestCase(unittest.TestCase): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() self.pruneLog() + self._log.close() logSys.handlers = self._old_handlers - logSys.level = self._old_level + logSys.setLevel(self._old_level) super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): From cbac7c176a93207027a53aa4591342f7268a1547 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Mar 2021 12:57:42 +0100 Subject: [PATCH 052/139] readline fixed to consider interim new-line character as part of code point in multi-byte logs (e. g. unicode: utf-16be, utf-16le); suppress warning "Error decoding line" for incomplete line (produced by not fully read multi-byte new-line character at end of data); added test coverage for such logs --- fail2ban/server/filter.py | 59 ++++++++++++++++++++++------- fail2ban/tests/filtertestcase.py | 64 +++++++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 23 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 845b069d..16279627 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1398,6 +1398,9 @@ class FileContainer: try: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: + # avoid warning if got incomplete end of line (e. g. '\n' in "...[0A" followed by "00]..." for utf-16le: + if (e.end == len(line) and line[e.start] in b'\r\n'): + return line[0:e.start].decode(enc, 'replace') global _decode_line_warn lev = 7 if not _decode_line_warn.get(filename, 0): @@ -1406,9 +1409,9 @@ class FileContainer: logSys.log(lev, "Error decoding line from '%s' with '%s'.", filename, enc) if logSys.getEffectiveLevel() <= lev: - logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", + logSys.log(lev, + "Consider setting logencoding to appropriate encoding for this jail. " + "Continuing to process line ignoring invalid characters: %r", line) # decode with replacing error chars: line = line.decode(enc, 'replace') @@ -1422,23 +1425,51 @@ class FileContainer: if line is not complete (and complete=True) or there is no content to read. If line is complete (and complete is True), it also shift current known position to begin of next line. + + Also it is safe against interim new-line bytes (e. g. part of multi-byte char) + in given encoding. """ if self.__handler is None: return "" - rl = self.__handler.readline() - if rl == b'': + # read raw bytes up to \n char: + b = self.__handler.readline() + if not b: return None - # trim new-line here and check the line was written complete (contains a new-line): - l = rl.rstrip(b'\r\n') - if self.inOperation and complete: - if l == rl: + bl = len(b) + # convert to log-encoding (new-line char could disappear if it is part of multi-byte sequence): + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if complete: + if l == r: + # try to fill buffer in order to find line-end in log encoding: + fnd = 0 + while 1: + r = self.__handler.readline() + if not r: + break + b += r + bl += len(r) + # convert to log-encoding: + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # ensure new-line is not in the middle (buffered 2 strings, e. g. in utf-16le it is "...[0A"+"00]..."): + e = r.find('\n') + if e >= 0 and e != len(r)-1: + l, r = r[0:e], r[0:e+1] + # back to bytes and get offset to seek after NL: + r = r.encode(self.getEncoding(), 'replace') + self.__handler.seek(-bl+len(r), 1) + return l + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if l != r: + return l # not fulfilled - seek back and return: - self.__handler.seek(self.__pos, 0) + self.__handler.seek(-bl, 1) return None - # shift position (to be able to seek back above): - self.__pos += len(rl) - return FileContainer.decode_line( - self.getFileName(), self.getEncoding(), l) + return l def close(self): if self.__handler is not None: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 799adfd3..f8621f29 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -644,6 +644,19 @@ class LogFile(LogCaptureTestCase): self.filter = FilterPoll(None) self.assertRaises(IOError, self.filter.addLogPath, LogFile.MISSING) + def testDecodeLineWarn(self): + # incomplete line (missing byte at end), warning is suppressed: + l = u"correct line\n" + r = l.encode('utf-16le') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) + self.assertNotLogged('Error decoding line') + # complete line (incorrect surrogate in the middle), warning is there: + r = b"incorrect \xc8\x0a line\n" + l = r.decode('utf-8', 'replace') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-8', r), l) + self.assertLogged('Error decoding line') + class LogFileFilterPoll(unittest.TestCase): @@ -1633,16 +1646,49 @@ class GetFailures(LogCaptureTestCase): def testCRLFFailures01(self): # We first adjust logfile/failures to end with CR+LF fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') - # poor man unix2dos: - fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') - for l in fin.read().splitlines(): - fout.write(l + b'\r\n') - fin.close() - fout.close() + try: + # poor man unix2dos: + fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') + for l in fin.read().splitlines(): + fout.write(l + b'\r\n') + fin.close() + fout.close() - # now see if we should be getting the "same" failures - self.testGetFailures01(filename=fname) - _killfile(fout, fname) + # now see if we should be getting the "same" failures + self.testGetFailures01(filename=fname) + finally: + _killfile(fout, fname) + + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + tm = int(time.time()) + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + ): + fout.write(l.encode(enc)) + fout.close() + + self.filter.setLogEncoding(enc) + self.filter.addLogPath(fname, autoSeek=0) + self.filter.setDatePattern((r'^EPOCH',)) + self.filter.addFailRegex(r"Failed .* from ") + self.filter.getFailures(fname) + self.assertLogged( + "[DummyJail] Found 192.0.2.1", + "[DummyJail] Found 192.0.2.2", all=True, wait=True) + finally: + _killfile(fout, fname) + self.filter.delLogPath(fname) + # must find 4 failures and generate 2 tickets (2 IPs with each 2 failures): + self.assertEqual(self.filter.failManager.getFailCount(), (2, 4)) def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, From 9659033523e27f62b2c6ad7db5e1671e58d4f045 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 16:19:06 +0100 Subject: [PATCH 053/139] fail2ban-regex: reimplemented log-file iterator - uses FileContainer facilities now instead of direct read from file and decode; fail2banregextestcase.py extended to cover proper line-ending handling by interim NL char as part of multi-byte encodings (utf-16be, utf-16le) --- fail2ban/client/fail2banregex.py | 13 ++++---- fail2ban/server/filter.py | 5 ++- fail2ban/tests/fail2banregextestcase.py | 44 +++++++++++++++++++++---- fail2ban/tests/filtertestcase.py | 2 +- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 90e178f9..26e7394a 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -289,9 +289,6 @@ class Fail2banRegex(object): def output(self, line): if not self._opts.out: output(line) - def decode_line(self, line): - return FileContainer.decode_line('', self._encoding, line) - def encode_line(self, line): return line.encode(self._encoding, 'ignore') @@ -724,8 +721,12 @@ class Fail2banRegex(object): return True def file_lines_gen(self, hdlr): - for line in hdlr: - yield self.decode_line(line) + while 1: + line = hdlr.readline() + if line is None: + break + yield line + hdlr.close() def start(self, args): @@ -745,7 +746,7 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = open(cmd_log, 'rb') + hdlr = FileContainer(cmd_log, self._encoding, doOpen=True) self.output( "Use log file : %s" % cmd_log ) self.output( "Use encoding : %s" % self._encoding ) test_lines = self.file_lines_gen(hdlr) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 16279627..9a2df255 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1278,7 +1278,7 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail=False): + def __init__(self, filename, encoding, tail=False, doOpen=False): self.__filename = filename self.setEncoding(encoding) self.__tail = tail @@ -1289,6 +1289,9 @@ class FileContainer: self.__hashNextTime = time.time() + 30 # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') + if doOpen: # fail2ban-regex only (don't need to reopen it and check for rotation) + self.__handler = handler + return try: stats = os.fstat(handler.fileno()) self.__ino = stats.st_ino diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 0a33fd9d..32acc0ae 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -25,6 +25,7 @@ __license__ = "GPL" import os import sys +import tempfile import unittest from ..client import fail2banregex @@ -80,6 +81,11 @@ def _test_exec_command_line(*args): sys.stderr = _org['stderr'] return _exit_code +def _reset(): + # reset global warn-counter: + from ..server.filter import _decode_line_warn + _decode_line_warn.clear() + STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" @@ -122,6 +128,7 @@ class Fail2banRegexTest(LogCaptureTestCase): """Call before every test case.""" LogCaptureTestCase.setUp(self) setUpMyTime() + _reset() def tearDown(self): """Call after every test case.""" @@ -454,14 +461,8 @@ class Fail2banRegexTest(LogCaptureTestCase): FILENAME_ZZZ_GEN, FILENAME_ZZZ_GEN )) - def _reset(self): - # reset global warn-counter: - from ..server.filter import _decode_line_warn - _decode_line_warn.clear() - def testWronChar(self): unittest.F2B.SkipIfCfgMissing(stock=True) - self._reset() self.assertTrue(_test_exec( "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", @@ -477,7 +478,6 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronCharDebuggex(self): unittest.F2B.SkipIfCfgMissing(stock=True) - self._reset() self.assertTrue(_test_exec( "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", @@ -490,6 +490,36 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + ): + fout.write(l.encode(enc)) + fout.close() + + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "--encoding", enc, + "--datepattern", r"^EPOCH", + fname, r"Failed .* from ", + )) + + self.assertLogged(" encoding : %s" % enc, + "Lines: 2 lines, 0 ignored, 2 matched, 0 missed", all=True) + self.assertNotLogged("Missed line(s)") + finally: + fout.close() + os.unlink(fname) + def testExecCmdLine_Usage(self): self.assertNotEqual(_test_exec_command_line(), 0) self.pruneLog() diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index f8621f29..319f16de 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1660,7 +1660,7 @@ class GetFailures(LogCaptureTestCase): _killfile(fout, fname) def testNLCharAsPartOfUniChar(self): - fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): for enc in ('utf-16be', 'utf-16le'): self.pruneLog("[test-phase encoding=%s]" % enc) From ccf4f3a07dbe5e21dbca1e0f3e702cbdaaeabaf3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 17:22:05 +0100 Subject: [PATCH 054/139] amend with common log-file iterator in fail2ban-regex and test-suite (in sample regex factory also) --- fail2ban/client/fail2banregex.py | 12 ++---------- fail2ban/server/filter.py | 17 ++++++++++++++--- fail2ban/tests/samplestestcase.py | 15 ++++++++------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 26e7394a..5921dfdd 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -720,14 +720,6 @@ class Fail2banRegex(object): return True - def file_lines_gen(self, hdlr): - while 1: - line = hdlr.readline() - if line is None: - break - yield line - hdlr.close() - def start(self, args): cmd_log, cmd_regex = args[:2] @@ -746,10 +738,10 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = FileContainer(cmd_log, self._encoding, doOpen=True) + test_lines = FileContainer(cmd_log, self._encoding, doOpen=True) + self.output( "Use log file : %s" % cmd_log ) self.output( "Use encoding : %s" % self._encoding ) - test_lines = self.file_lines_gen(hdlr) except IOError as e: # pragma: no cover output( e ) return False diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9a2df255..e16d86c9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1280,6 +1280,7 @@ class FileContainer: def __init__(self, filename, encoding, tail=False, doOpen=False): self.__filename = filename + self.waitForLineEnd = True self.setEncoding(encoding) self.__tail = tail self.__handler = None @@ -1469,9 +1470,10 @@ class FileContainer: l = r.rstrip('\r\n') if l != r: return l - # not fulfilled - seek back and return: - self.__handler.seek(-bl, 1) - return None + if self.waitForLineEnd: + # not fulfilled - seek back and return: + self.__handler.seek(-bl, 1) + return None return l def close(self): @@ -1482,6 +1484,15 @@ class FileContainer: self.__handler.close() self.__handler = None + def __iter__(self): + return self + def next(self): + line = self.readline() + if line is None: + self.close() + raise StopIteration + return line + _decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 5a72ffa9..b33b46c1 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -23,7 +23,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import datetime -import fileinput import inspect import json import os @@ -156,12 +155,15 @@ def testSampleRegexsFactory(name, basedir): i = 0 while i < len(filenames): filename = filenames[i]; i += 1; - logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", - filename), mode='rb') + logFile = FileContainer(os.path.join(TEST_FILES_DIR, "logs", + filename), 'UTF-8', doOpen=True) + # avoid errors if no NL char at end of test log-file: + logFile.waitForLineEnd = False ignoreBlock = False + lnnum = 0 for line in logFile: - line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) + lnnum += 1 jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: try: @@ -201,9 +203,8 @@ def testSampleRegexsFactory(name, basedir): # failJSON - faildata contains info of the failure to check it. except ValueError as e: # pragma: no cover - we've valid json's raise ValueError("%s: %s:%i" % - (e, logFile.filename(), logFile.filelineno())) + (e, logFile.getFileName(), lnnum)) line = next(logFile) - line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable @@ -298,7 +299,7 @@ def testSampleRegexsFactory(name, basedir): import pprint raise AssertionError("%s: %s on: %s:%i, line:\n %s\nregex (%s):\n %s\n" "faildata: %s\nfail: %s" % ( - fltName, e, logFile.filename(), logFile.filelineno(), + fltName, e, logFile.getFileName(), lnnum, line, failregex, regexList[failregex] if failregex != -1 else None, '\n'.join(pprint.pformat(faildata).splitlines()), '\n'.join(pprint.pformat(fail).splitlines()))) From 4b17dddc2389a71bba15be9a3129b03665f853b0 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 25 Mar 2021 12:05:09 +0100 Subject: [PATCH 055/139] update ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index e5e7b485..4fe8acdc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,12 +18,17 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition different from 0) in case of unsane environment. ### Fixes +* readline fixed to consider interim new-line character as part of code point in multi-byte logs + (e. g. unicode encoding like utf-16be, utf-16le); * `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) ### New Features and Enhancements * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) +* better recognition of log rotation, better performance by reopen: avoid unnecessary seek to begin of file + (and hash calculation) +* file filter reads only complete lines (ended with new-line) now, so waits for end of line (for its completion) ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools From b259e819114e587e0e9b17253860dd48f9579997 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 17:23:14 +0100 Subject: [PATCH 056/139] test-suite: skip testFQDN if no network --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 319f16de..27f5695e 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -2252,6 +2252,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2)) def testFQDN(self): + unittest.F2B.SkipIfNoNetwork() sname = DNSUtils.getHostname(fqdn=False) lname = DNSUtils.getHostname(fqdn=True) # FQDN is not localhost if short hostname is not localhost too (or vice versa): From 2686811593a57bab72f691a5c1c93ffb877d4395 Mon Sep 17 00:00:00 2001 From: j-marz Date: Sun, 28 Mar 2021 21:19:10 +1100 Subject: [PATCH 057/139] Updated zoneminder filter Support new log format, ERR instead of WAR. Add detection of non-existent user login attempts --- config/filter.d/zoneminder.conf | 16 +++++++++++----- fail2ban/tests/files/logs/zoneminder | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf index cc82755a..1af97c7d 100644 --- a/config/filter.d/zoneminder.conf +++ b/config/filter.d/zoneminder.conf @@ -5,17 +5,23 @@ before = apache-common.conf [Definition] -# pattern: [Wed Apr 27 23:12:07.736196 2016] [:error] [pid 2460] [client 10.1.1.1:47296] WAR [Login denied for user "test"], referer: https://zoneminderurl/index.php -# -# +# patterns: + # [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ + # [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ + # [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ + # Option: failregex -# Notes.: regex to match the password failure messages in the logfile. +# Notes.: regex to match the login failure and non-existent user error messages in the logfile. failregex = ^%(_apache_error_client)s WAR \[Login denied for user "[^"]*"\] + ^%(_apache_error_client)s ERR \[Login denied for user "[^"]*"\] + ^%(_apache_error_client)s ERR \[Could not retrieve user \w* details\] ignoreregex = # Notes: -# Tested on Zoneminder 1.29.0 +# Tested on Zoneminder 1.29 and 1.35.21 +# +# Zoneminer versions > 1.3x use "ERR" and < 1.3x use "WAR" level logs, so i've kept both for compatibility reasons # # Author: John Marzella diff --git a/fail2ban/tests/files/logs/zoneminder b/fail2ban/tests/files/logs/zoneminder index abd49869..f4b6bd3e 100644 --- a/fail2ban/tests/files/logs/zoneminder +++ b/fail2ban/tests/files/logs/zoneminder @@ -1,2 +1,8 @@ # failJSON: { "time": "2016-03-28T16:50:49", "match": true , "host": "10.1.1.1" } [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ + +# failJSON: { "time": "2021-03-28T16:53:00", "match": true , "host": "10.1.1.1" } +[Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user username1 details], referer: https://zm/zm/?view=logout + +# failJSON: { "time": "2021-03-28T16:59:14", "match": true , "host": "10.1.1.1" } +[Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "username1"], referer: https://zm/zm/? From 5d8f5004718d38146c6821f37eaeb3c7b0415d9d Mon Sep 17 00:00:00 2001 From: j-marz Date: Mon, 29 Mar 2021 08:36:53 +1100 Subject: [PATCH 058/139] updated formatting to pass tests --- config/filter.d/zoneminder.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf index 1af97c7d..7da40968 100644 --- a/config/filter.d/zoneminder.conf +++ b/config/filter.d/zoneminder.conf @@ -5,11 +5,10 @@ before = apache-common.conf [Definition] -# patterns: - # [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ - # [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ - # [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ - +# patterns: [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ +# [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ +# [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ +# # Option: failregex # Notes.: regex to match the login failure and non-existent user error messages in the logfile. From 5aa20c30d8b283970e4b3ee0938f729850ebb618 Mon Sep 17 00:00:00 2001 From: Markus Felten Date: Fri, 12 Feb 2021 09:29:03 +0100 Subject: [PATCH 059/139] fix: add journalmatch to nginx filters --- config/filter.d/nginx-bad-request.conf | 2 ++ config/filter.d/nginx-botsearch.conf | 4 +++- config/filter.d/nginx-http-auth.conf | 2 ++ config/filter.d/nginx-limit-req.conf | 3 +++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index 2b8f5ab6..12c14ab7 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -11,4 +11,6 @@ datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,] ^[^\[]*\[({DATE}) {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # Author: Jan Przybylak diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 0be895b2..2bd23072 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -17,7 +17,9 @@ datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,] ^[^\[]*\[({DATE}) {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV Notes: # Based on apache-botsearch filter # -# Author: Frantisek Sumsal \ No newline at end of file +# Author: Frantisek Sumsal diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 93341cd2..3a2e314b 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -10,6 +10,8 @@ ignoreregex = datepattern = {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV NOTES: # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Extensive search of all nginx auth failures not done yet. diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index e23548ab..2f45e831 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -44,3 +44,6 @@ failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by ignoreregex = datepattern = {^LN-BEG} + +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + From 9eaa2322b0913c805cfa7d45be96d0067ffe61b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Olschewsky?= Date: Mon, 24 Feb 2020 16:35:42 +0100 Subject: [PATCH 060/139] Filter and Defaults for Microsoft SQL Server --- config/filter.d/mssql-auth.conf | 15 +++++++++++++++ config/jail.conf | 8 ++++++++ fail2ban/tests/files/logs/mssql-auth | 3 +++ 3 files changed, 26 insertions(+) create mode 100644 config/filter.d/mssql-auth.conf create mode 100644 fail2ban/tests/files/logs/mssql-auth diff --git a/config/filter.d/mssql-auth.conf b/config/filter.d/mssql-auth.conf new file mode 100644 index 00000000..a1813c83 --- /dev/null +++ b/config/filter.d/mssql-auth.conf @@ -0,0 +1,15 @@ +# Fail2Ban filter for failed MSSQL Server authentication attempts + +[Definition] + +failregex = Logon\s+Login failed for user ('.*')(.*)\[CLIENT: \]$ + + +# DEV Notes: +# Tested with SQL Server 2019 on Ubuntu 18.04 +# +# Example: +# 2020-02-24 14:48:55.12 Logon Login failed for user 'root'. Reason: Could not find a login matching the name provided. [CLIENT: 127.0.0.1] +# +# Author: Rüdiger Olschewsky +# \ No newline at end of file diff --git a/config/jail.conf b/config/jail.conf index 20958d11..aeff3616 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -786,6 +786,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation diff --git a/fail2ban/tests/files/logs/mssql-auth b/fail2ban/tests/files/logs/mssql-auth new file mode 100644 index 00000000..5a97f53e --- /dev/null +++ b/fail2ban/tests/files/logs/mssql-auth @@ -0,0 +1,3 @@ +2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 212.96.131.253] +2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 148.86.203.199] +2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 105.254.136.171] \ No newline at end of file From 7f38b80d351cc51cd3273ad93cd1939a4b1abe1e Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 3 Apr 2021 20:16:47 +0200 Subject: [PATCH 061/139] precise regex (left anchor and fewer catch-all's); fixed tests (added failJSON and more tests for some corner-cases around new RE) --- config/filter.d/mssql-auth.conf | 2 +- fail2ban/tests/files/logs/mssql-auth | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config/filter.d/mssql-auth.conf b/config/filter.d/mssql-auth.conf index a1813c83..65bbd917 100644 --- a/config/filter.d/mssql-auth.conf +++ b/config/filter.d/mssql-auth.conf @@ -2,7 +2,7 @@ [Definition] -failregex = Logon\s+Login failed for user ('.*')(.*)\[CLIENT: \]$ +failregex = ^\s*Logon\s+Login failed for user '(?:[^']*|.*)'\. [^'\[]+\[CLIENT: \]$ # DEV Notes: diff --git a/fail2ban/tests/files/logs/mssql-auth b/fail2ban/tests/files/logs/mssql-auth index 5a97f53e..1c9b65ec 100644 --- a/fail2ban/tests/files/logs/mssql-auth +++ b/fail2ban/tests/files/logs/mssql-auth @@ -1,3 +1,11 @@ -2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 212.96.131.253] -2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 148.86.203.199] -2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 105.254.136.171] \ No newline at end of file +# failJSON: { "time": "2020-02-24T16:05:21", "match": true , "host": "192.0.2.1" } +2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.1] +# failJSON: { "time": "2020-02-24T16:30:25", "match": true , "host": "192.0.2.2" } +2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.2] +# failJSON: { "time": "2020-02-24T16:31:12", "match": true , "host": "192.0.2.3" } +2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 192.0.2.3] + +# failJSON: { "time": "2020-02-24T16:31:26", "match": true , "host": "192.0.2.4", "user":"O'Leary" } +2020-02-24 16:31:26.01 Logon Login failed for user 'O'Leary'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.4] +# failJSON: { "time": "2020-02-24T16:31:26", "match": false, "desc": "test injection in possibly unescaped foreign input" } +2020-02-24 16:31:26.02 Wrong data received: Logon Login failed for user 'test'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.5] From 1215cb28ac6a79acda2699694556bc1a6cef1e45 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 20:58:26 +0200 Subject: [PATCH 062/139] Update nsd --- fail2ban/tests/files/logs/nsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/nsd b/fail2ban/tests/files/logs/nsd index a33a52a9..63c162e9 100644 --- a/fail2ban/tests/files/logs/nsd +++ b/fail2ban/tests/files/logs/nsd @@ -2,3 +2,5 @@ [1387288694] nsd[7745]: info: ratelimit block example.com. type any target 192.0.2.0/24 query 192.0.2.105 TYPE255 # failJSON: { "time": "2013-12-18T07:42:15", "match": true , "host": "192.0.2.115" } [1387348935] nsd[23600]: info: axfr for zone domain.nl. from client 192.0.2.115 refused, no acl matches. +# failJSON: { "time": "2021-03-05T05:25:14", "match": true , "host": "192.0.2.32", "desc": "new format, no client after from, no dot at end, gh-2965" } +[2021-03-05 05:25:14.562] nsd[160800]: info: axfr for example.com. from 192.0.2.32 refused, no acl matches From a838deba7f8066d50984ed24983fb90f15edfaaf Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 21:00:14 +0200 Subject: [PATCH 063/139] restore anchor (e. g. catch all in the middle), dot is optional now, RE rewritten a bit more precise --- config/filter.d/nsd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index 9399db23..0589c16c 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -22,8 +22,8 @@ _daemon = nsd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ - ^%(__prefix_line)sinfo: .* refused, no acl matches +failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ + ^%(__prefix_line)sinfo: .* from(?: client)? refused, no acl matches\.?$ ignoreregex = From 0c4d356d118d1282d8bdbed21cb514f02953e65f Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:10:51 +0200 Subject: [PATCH 064/139] added test log-file --- fail2ban/tests/files/logs/scanlogd | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 fail2ban/tests/files/logs/scanlogd diff --git a/fail2ban/tests/files/logs/scanlogd b/fail2ban/tests/files/logs/scanlogd new file mode 100644 index 00000000..5a97c578 --- /dev/null +++ b/fail2ban/tests/files/logs/scanlogd @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-03-05T21:44:43", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:43 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 80, 81, 83, 88, 99, 443, 1080, 3128, ..., f????uxy, TOS 00, TTL 49 @20:44:43 +# failJSON: { "time": "2005-03-05T21:44:44", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:44 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 497, 515, 544, 543, 464, 513, ..., fSrpauxy, TOS 00 @09:04:25 +# failJSON: { "time": "2005-03-05T21:44:45", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:45 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 593, 548, 636, 646, 625, 631, ..., fSrpauxy, TOS 00, TTL 239 @17:34:00 +# failJSON: { "time": "2005-03-05T21:44:46", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:46 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 22, 26, 37, 80, 25, 79, ..., fSrpauxy, TOS 00 @22:38:37 From 14edeed310ea5d74a13ec72d415b934cb4f7beb1 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:24:55 +0200 Subject: [PATCH 065/139] fixed regex (don't need to match whole line, e. g. every port etc) --- config/filter.d/scanlogd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf index 65ad63f6..15fea329 100644 --- a/config/filter.d/scanlogd.conf +++ b/config/filter.d/scanlogd.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = scanlogd -failregex = ^%(__prefix_line)s\ to\ [\.:0-9a-f]+\ ports\ [\ \.,0-9]+,\ f.......,\ TOS\ [0-9]+,\ TTL\ [0-9]+\ \@[0-9]{1,2}:[0-9]{2}:[0-9]{2}$ +failregex = ^%(__prefix_line)s(?::) to \S+ ports\b ignoreregex = From 977dfe4bd762474e4e2d077e258c49772a48f6f3 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:29:16 +0200 Subject: [PATCH 066/139] small amend: sport after saddr is optional format of message: saddr[:sport] to daddr [and others,] ports port[, port...], ..., flags[, TOS TOS][, TTL TTL] @HH:MM:SS --- config/filter.d/scanlogd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf index 15fea329..d3fe78b0 100644 --- a/config/filter.d/scanlogd.conf +++ b/config/filter.d/scanlogd.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = scanlogd -failregex = ^%(__prefix_line)s(?::) to \S+ ports\b +failregex = ^%(__prefix_line)s(?::)? to \S+ ports\b ignoreregex = From 2d51240b3e9e19a4259c03058ddde6ce8698397b Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:33:49 +0200 Subject: [PATCH 067/139] correction for default log interpolation and added allports banaction --- config/jail.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index d6d8af67..52cc161d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -967,4 +967,5 @@ port = http,https logpath = /var/log/traefik/access.log [scanlogd] -logpath = %{syslog_local0} +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s From 7579072e3b1d0663ac50ff067a953d4fe8c28ac7 Mon Sep 17 00:00:00 2001 From: Michele Mondelli Date: Thu, 31 Dec 2020 17:03:19 +0100 Subject: [PATCH 068/139] docs: fix typos --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index b4b9f249..cd802a57 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -67,7 +67,7 @@ before = paths-debian.conf # more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : #bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) -# "bantime.multipliers" used to calculate next value of ban time instead of formula, coresponding +# "bantime.multipliers" used to calculate next value of ban time instead of formula, corresponding # previously ban count and given "bantime.factor" (for multipliers default is 1); # following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, # always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours @@ -77,7 +77,7 @@ before = paths-debian.conf #bantime.multipliers = 1 5 30 60 300 720 1440 2880 # "bantime.overalljails" (if true) specifies the search of IP in the database will be executed -# cross over all jails, if false (dafault), only current jail of the ban IP will be searched +# cross over all jails, if false (default), only current jail of the ban IP will be searched #bantime.overalljails = false # -------------------- From c5d43d7573437e401f991a782e106a851ef9975d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 4 Apr 2021 00:00:59 +0200 Subject: [PATCH 069/139] Update ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index f9fb8a33..56b8c757 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,9 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) +* `filter.d/nginx-http-auth.conf` - extended with parameter mode, so additionally to `auth` (or `normal`) + mode `fallback` (or combined as `aggressive`) can find SSL errors while SSL handshaking, gh-2881 + ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools ----------- From b2f6a3a658f83fa6e2ec80deffa44c837775e80a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 4 Apr 2021 00:21:59 +0200 Subject: [PATCH 070/139] remove unneeded substitution it is enough to add `apprise` to action --- config/jail.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index dbca2e8f..e17a1892 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -231,10 +231,10 @@ action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(prot # See https://github.com/caronc/apprise/wiki for details on what is supported. # # You may optionally over-ride the default configuration line (containing the Apprise URLs) -# by using 'apprise[name=%(__name__)s, config="/alternate/path/to/apprise.cfg"]' otherwise +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise # /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. -action_apprise = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - apprise[name=%(__name__)s] +# action = %(action_)s +# apprise # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. From 8b741129a51c98b6178f08bcca9af88a4486201f Mon Sep 17 00:00:00 2001 From: Jordi Sanfeliu Date: Wed, 14 Apr 2021 11:26:52 +0200 Subject: [PATCH 071/139] Create monitorix-httpd --- fail2ban/tests/files/logs/monitorix-httpd | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 fail2ban/tests/files/logs/monitorix-httpd diff --git a/fail2ban/tests/files/logs/monitorix-httpd b/fail2ban/tests/files/logs/monitorix-httpd new file mode 100644 index 00000000..3166dbd3 --- /dev/null +++ b/fail2ban/tests/files/logs/monitorix-httpd @@ -0,0 +1,10 @@ +Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix-cgi/monitorix.cgi - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix/css/black.css - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs01.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs03.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs04.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs02.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +Wed Apr 14 08:54:22 2021 - NOTEXIST - [127.0.0.1] File does not exist: /manager/html +Wed Apr 14 11:24:31 2021 - NOTALLOWED - [127.0.0.1] Access not allowed: /monitorix/ +Wed Apr 14 11:26:08 2021 - AUTHERR - [127.0.0.1] Authentication error: /monitorix/ +Wed Apr 14 11:26:09 2021 - AUTHERR - [127.0.0.1] Authentication error: /monitorix/ From 63b3f39adc7323873d7f558a6060f79ce3e9a273 Mon Sep 17 00:00:00 2001 From: Jordi Sanfeliu Date: Wed, 14 Apr 2021 11:30:48 +0200 Subject: [PATCH 072/139] Rename monitorix-httpd to monitorix --- fail2ban/tests/files/logs/{monitorix-httpd => monitorix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fail2ban/tests/files/logs/{monitorix-httpd => monitorix} (100%) diff --git a/fail2ban/tests/files/logs/monitorix-httpd b/fail2ban/tests/files/logs/monitorix similarity index 100% rename from fail2ban/tests/files/logs/monitorix-httpd rename to fail2ban/tests/files/logs/monitorix From b6fac90b5a57934749f30a0f9c626b075fcf51d8 Mon Sep 17 00:00:00 2001 From: Jordi Sanfeliu Date: Wed, 14 Apr 2021 11:46:23 +0200 Subject: [PATCH 073/139] Update monitorix --- fail2ban/tests/files/logs/monitorix | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/files/logs/monitorix b/fail2ban/tests/files/logs/monitorix index 3166dbd3..e6ad6dc6 100644 --- a/fail2ban/tests/files/logs/monitorix +++ b/fail2ban/tests/files/logs/monitorix @@ -1,10 +1,8 @@ +# failJSON: { "time": "2021-04-14T08:11:01", "match": false, "desc": "should be ignored: successful request" } Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix-cgi/monitorix.cgi - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" -Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix/css/black.css - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" -Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs01.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" -Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs03.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" -Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs04.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" -Wed Apr 14 08:11:02 2021 - OK - [127.0.0.1] "GET /monitorix/imgs/fs02.1day.png - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +# failJSON: { "time": "2021-04-14T08:54:22", "match": true, "host": "127.0.0.1", "desc": "file does not exist" } Wed Apr 14 08:54:22 2021 - NOTEXIST - [127.0.0.1] File does not exist: /manager/html +# failJSON: { "time": "2021-04-14T11:24:31", "match": true, "host": "127.0.0.1", "desc": "access not allowed" } Wed Apr 14 11:24:31 2021 - NOTALLOWED - [127.0.0.1] Access not allowed: /monitorix/ +# failJSON: { "time": "2021-04-14T11:26:08", "match": true, "host": "127.0.0.1", "desc": "authentication error" } Wed Apr 14 11:26:08 2021 - AUTHERR - [127.0.0.1] Authentication error: /monitorix/ -Wed Apr 14 11:26:09 2021 - AUTHERR - [127.0.0.1] Authentication error: /monitorix/ From ab0847e2d59611dd81e2f6a1dd30c1f136dce958 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 14 Apr 2021 13:06:58 +0200 Subject: [PATCH 074/139] more precise anchored RE (also combining all 3 REs in a single regex) --- config/filter.d/monitorix.conf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/filter.d/monitorix.conf b/config/filter.d/monitorix.conf index 3979ed43..ff69f1bc 100644 --- a/config/filter.d/monitorix.conf +++ b/config/filter.d/monitorix.conf @@ -16,9 +16,7 @@ _daemon = monitorix-httpd # (?:::f{4,6}:)?(?P\S+) # Values: TEXT # -failregex = NOTEXIST - \[\] .* - AUTHERR - \[\] .* - NOTALLOWED - \[\] .* +failregex = ^(?:\s+-)?\s*(?:NOTEXIST|AUTHERR|NOTALLOWED) - \b # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From f0214b3d36ea08b318c52a2d51fce50bf9ec0cce Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Apr 2021 18:13:40 +0200 Subject: [PATCH 075/139] filter.d/sendmail-reject.conf: fixed regex to consider "Connection rate limit exceeded" with different combination of arguments --- config/filter.d/sendmail-reject.conf | 4 ++-- fail2ban/tests/files/logs/sendmail-reject | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index e8b766c5..966d880c 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,12 +21,12 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? -addr = (?:IPv6:|) +addr = (?:(?:IPv6:)?|) prefregex = ^%(__prefix_line)s.+$ cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index 99c1877c..ed3143a8 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -40,6 +40,9 @@ Feb 27 15:49:07 batman sm-mta[88390]: ruleset=check_relay, arg1=189-30-205-74.pa # failJSON: { "time": "2005-02-19T18:01:50", "match": true , "host": "196.213.73.146" } Feb 19 18:01:50 batman sm-mta[78152]: ruleset=check_relay, arg1=[196.213.73.146], arg2=196.213.73.146, relay=[196.213.73.146], reject=421 4.3.2 Connection rate limit exceeded. +# failJSON: { "time": "2005-02-19T20:17:12", "match": true , "host": "192.0.2.123" } +Feb 19 20:17:12 server sm-mta[201892]: ruleset=check_relay, arg1=[192.0.2.123], arg2=192.0.2.123, relay=host.example.com [192.0.2.123] (may be forged), reject=421 4.3.2 Connection rate limit exceeded. + # failJSON: { "time": "2005-02-27T10:53:06", "match": true , "host": "209.15.212.253" } Feb 27 10:53:06 batman sm-mta[44307]: s1R9r60D044307: rejecting commands from [209.15.212.253] due to pre-greeting traffic after 0 seconds # failJSON: { "time": "2005-02-27T10:53:07", "match": true , "host": "1.2.3.4" } From d3f5d2d52b1e1b1577dbf29e931fbdbe59f56109 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 21 Apr 2021 11:50:07 +0200 Subject: [PATCH 076/139] documentation (interpolation tags) --- man/jail.conf.5 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 788fad2b..5f29161d 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -481,13 +481,29 @@ is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. T .IP \fI\fR - regex for IPv4 addresses. .IP -\fI\fR - regex for IPv6 addresses (also IP enclosed in brackets). +\fI\fR - regex for IPv6 addresses. .IP \fI\fR - regex to match hostnames. .IP \fI\fR - helper regex to match CIDR (simple integer form of net-mask). .IP \fI\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +.IP +\fI...\fR - free regex capturing group targeting identifier used for ban (instead of IP address or hostname). +.IP +\fI...\fR - free regex capturing named group stored in ticket, which can be used in action. +.nf +For example \fI[^@]+\fR matches and stores a user name, that can be used in action with interpolation tag \fI\fR. +.IP +\fI...\fR - free regex capturing alternative named group stored in ticket. +.nf +For example first found matched value defined in regex as \fI\fR, \fI\fR or \fI\fR would be stored as (if direct match is not found or empty). +.PP +Every of abovementioned tags can be specified in \fBprefregex\fR and in \fBfailregex\fR, thereby if specified in both, the value matched in \fBfailregex\fR overwrites a value matched in \fBprefregex\fR. +.TQ +All standard tags like IP4 or IP6 can be also specified with custom regex using \fI...\fR syntax, for example \fI(?:ip4:\\S+|ip6:\\S+)\fR. +.TQ +Tags \fI\fR, \fI\fR and \fI\fR would also match the IP address enclosed in square brackets. .PP \fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message). .PP From 319cfefac2b57c9ca7c9e4ce91b252dd7e20cfee Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 27 Apr 2021 13:34:22 +0200 Subject: [PATCH 077/139] fix travis build (unsupported pythons and pypy versions), update 3.10 in GH actions --- .github/workflows/main.yml | 2 +- .travis.yml | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 262448c2..4ea0e7eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-alpha.5', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-beta.1', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/.travis.yml b/.travis.yml index 064b678b..398c120a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,16 +10,8 @@ dist: xenial matrix: fast_finish: true include: - - python: 2.6 - dist: trusty # required for Python 2.6 - python: 2.7 - dist: trusty # required for packages like gamin - name: 2.7 (trusty) - - python: 2.7 - name: 2.7 (xenial) - - python: pypy - - python: 3.3 - dist: trusty + #- python: pypy - python: 3.4 - python: 3.5 - python: 3.6 From e4e7a83cffb4fb5cbb35c204795fba45c8f40c61 Mon Sep 17 00:00:00 2001 From: usernamepi <53445688+usernamepi@users.noreply.github.com> Date: Thu, 6 May 2021 13:44:36 +0200 Subject: [PATCH 078/139] Update ufw.conf Prerequisites: * The ss command is available, kernel is compiled with option CONFIG_INET_DIAG_DESTROY. * Ufw version is => 0.36 (released in 2018) * Now using "prepend" instead of "insert" to be able to handle IPv6 addresses correctly. The current action will fail for IPv6 addresses. * Now application names containing a space should handled correctly, solves https://github.com/fail2ban/fail2ban/pull/1532 * Now closing IPv4 and IPv6 connections (if any) from the ip that is being banned. The current action will leave them open. Using ss to accomplish this. For this to work the kernel needs to be compiled with the CONFIG_INET_DIAG_DESTROY option. My system apparently is compiled that way. --- config/action.d/ufw.conf | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index d2f731f2..b47fa7e7 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -13,17 +13,26 @@ actionstop = actioncheck = -actionban = [ -n "" ] && app="app " - ufw insert from to $app +# ufw does "quickly process packets for which we already have a connection" in before.rules, +# therefore all related sockets should be closed +# actionban is using `ss` to do so, this only handles IPv4 and IPv6. -actionunban = [ -n "" ] && app="app " - ufw delete from to $app +actionban = if [ -n "" ] && ufw app info "" + then + ufw prepend from to app "" comment "" + else + ufw prepend from to comment "" + fi + ss -K dst [] + +actionunban = if [ -n "" ] && ufw app info "" + then + ufw delete from to app "" + else + ufw delete from to + fi [Init] -# Option: insertpos -# Notes.: The position number in the firewall list to insert the block rule -insertpos = 1 - # Option: blocktype # Notes.: reject or deny blocktype = reject From 5debaa4cac2723fa863ede9ed32c19cc82c71786 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 May 2021 20:23:58 +0200 Subject: [PATCH 079/139] option "add", can be set to "insert " instead of prepend (customization or backwards compat) --- config/action.d/ufw.conf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index b47fa7e7..bf06fe67 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -19,9 +19,9 @@ actioncheck = actionban = if [ -n "" ] && ufw app info "" then - ufw prepend from to app "" comment "" + ufw from to app "" comment "" else - ufw prepend from to comment "" + ufw from to comment "" fi ss -K dst [] @@ -33,6 +33,10 @@ actionunban = if [ -n "" ] && ufw app info "" fi [Init] +# Option: add +# Notes.: can be set to "insert 1" to insert a rule at certain position (here 1): +add = prepend + # Option: blocktype # Notes.: reject or deny blocktype = reject From 8f6a8df3a45395620e434fd15b4ede694a1d00aa Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 May 2021 21:47:06 +0200 Subject: [PATCH 080/139] added new options `kill-mode` and `kill`, which makes the drop of all connections optional --- config/action.d/ufw.conf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index bf06fe67..cf8c22be 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -23,7 +23,7 @@ actionban = if [ -n "" ] && ufw app info "" else ufw from to comment "" fi - ss -K dst [] + actionunban = if [ -n "" ] && ufw app info "" then @@ -32,6 +32,21 @@ actionunban = if [ -n "" ] && ufw app info "" ufw delete from to fi +# Option: kill-mode +# Notes.: can be set to ss (may be extended later with other modes) to immediately drop all connections from banned IP, default empty (no kill) +# Example: banaction = ufw[kill-mode=ss] +kill-mode = + +# intern conditional parameter used to provide killing mode after ban: +_kill_ = +_kill_ss = ss -K dst "[]" + +# Option: kill +# Notes.: can be used to specify custom killing feature, by default depending on option kill-mode +# Examples: banaction = ufw[kill='ss -K "( sport = :http || sport = :https )" dst "[]"'] + banaction = ufw[kill='cutter ""'] +kill = <_kill_> + [Init] # Option: add # Notes.: can be set to "insert 1" to insert a rule at certain position (here 1): From 2958ad8636e9b3082c49537d7a56ae3b8cf12012 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 May 2021 22:19:38 +0200 Subject: [PATCH 081/139] Update ChangeLog --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4b0733d7..175d087a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,9 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition ### Fixes * readline fixed to consider interim new-line character as part of code point in multi-byte logs (e. g. unicode encoding like utf-16be, utf-16le); +* `action.d/ufw.conf`: + - fixed handling on IPv6 (using prepend, gh-2331, gh-3018) + - application names containing spaces can be used now (gh-656, gh-1532, gh-3018) * `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) ### New Features and Enhancements @@ -29,6 +32,9 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * better recognition of log rotation, better performance by reopen: avoid unnecessary seek to begin of file (and hash calculation) * file filter reads only complete lines (ended with new-line) now, so waits for end of line (for its completion) +* `action.d/ufw.conf` (gh-3018): + - new option `add` (default `prepend`), can be supplied as `insert 1` for ufw versions before v.0.36 (gh-2331, gh-3018) + - new options `kill-mode` and `kill` to drop established connections of intruder (see action for details, gh-3018) * `filter.d/nginx-http-auth.conf` - extended with parameter mode, so additionally to `auth` (or `normal`) mode `fallback` (or combined as `aggressive`) can find SSL errors while SSL handshaking, gh-2881 From 88f779ed241e2fccbaeb65281d336ff0f1706b07 Mon Sep 17 00:00:00 2001 From: usernamepi <53445688+usernamepi@users.noreply.github.com> Date: Thu, 6 May 2021 23:23:39 +0200 Subject: [PATCH 082/139] ufw.conf, amend to #3018 - add missing option for comment (#3019) --- config/action.d/ufw.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index cf8c22be..b5d1213c 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -64,6 +64,10 @@ destination = any # Notes.: application from sudo ufw app list application = +# Option: comment +# Notes.: comment for rule added by fail2ban +comment = by Fail2Ban after attempts against + # DEV NOTES: # # Author: Guilhem Lettron From 2918849f9e781fc27a0fd0ba052c1a97d96d02bc Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 May 2021 01:10:26 +0200 Subject: [PATCH 083/139] fixes precise year pattern %ExY - accept years 20xx up to current century (using almost the same pattern in tests and production now) --- fail2ban/server/strptime.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 69514b20..6f88add1 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -104,14 +104,14 @@ def _updateTimeRE(): # more precise year patterns, within same century of last year and # the next 3 years (for possible long uptime of fail2ban); thereby - # respect possible run in the test-cases (alternate date used there): - if MyTime.alternateNowTime != 0: - timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) - timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) - else: # accept years: 19xx|2xxx up to current century - timeRE['ExY'] = r"(?P(?:19\d{2}|%s\d))" % _getYearCentRE(cent=(0,3), distance=3, - now=(MyTime.now(), datetime.datetime.fromtimestamp(978393600))) - timeRE['Exy'] = r"(?P\d{2})" + # consider possible run in the test-cases (alternate date used there), + # so accept years: 20xx (from test-date or 2001 up to current century) + timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3, + now=(datetime.datetime.now(), datetime.datetime.fromtimestamp( + min(MyTime.alternateNowTime or 978393600, 978393600)) + ) + ) + timeRE['Exy'] = r"(?P\d{2})" _updateTimeRE() From ef5c826c74c160d33d4b79b23d564f944d16a9dd Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 May 2021 01:16:48 +0200 Subject: [PATCH 084/139] fixes search for the best datepattern (gh-3020) - e. g. if line is too short, boundaries check for previously known unprecise pattern may fail on incomplete lines (logging break-off, no flush, etc) --- fail2ban/server/datedetector.py | 4 ++-- fail2ban/tests/fail2banregextestcase.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index ecc9d935..b90e1b26 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -365,10 +365,10 @@ class DateDetector(object): # with space or some special char), otherwise possible collision/pattern switch: if (( line[distance-1:distance] == self.__lastPos[1] or - (line[distance] == self.__lastPos[2] and not self.__lastPos[2].isalnum()) + (line[distance:distance+1] == self.__lastPos[2] and not self.__lastPos[2].isalnum()) ) and ( line[endpos:endpos+1] == self.__lastEndPos[2] or - (line[endpos-1] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum()) + (line[endpos-1:endpos] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum()) )): # search in line part only: log(logLevel-1, " boundaries are correct, search in part %r", line[distance:endpos]) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 85fe4f15..1c55e227 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -384,7 +384,17 @@ class Fail2banRegexTest(LogCaptureTestCase): "Found a match but no valid date/time found", "Match without a timestamp:", all=True) - self.pruneLog() + def testIncompleteDateTime(self): + # datepattern in followed lines doesn't match previously known pattern + line is too short + # (logging break-off, no flush, etc): + self.assertTrue(_test_exec( + '-o', 'Found-ADDR:', + '192.0.2.1 - - [02/May/2021:18:40:55 +0100] "GET / HTTP/1.1" 302 328 "-" "Mozilla/5.0" "-"\n' + '192.0.2.2 - - [02/May/2021:18:40:55 +0100\n' + '192.0.2.3 - - [02/May/2021:18:40:55', + '^')) + self.assertLogged( + "Found-ADDR:192.0.2.1", "Found-ADDR:192.0.2.2", "Found-ADDR:192.0.2.3", all=True) def testFrmtOutputWrapML(self): unittest.F2B.SkipIfCfgMissing(stock=True) From 4f8427178a805fe7388c5fc8cfb16e2e601af920 Mon Sep 17 00:00:00 2001 From: usernamepi <53445688+usernamepi@users.noreply.github.com> Date: Fri, 7 May 2021 18:23:40 +0200 Subject: [PATCH 085/139] Missing comment "#" (#3022) Missed this ... but the logs showed it. --- config/action.d/ufw.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index b5d1213c..088a5a9d 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -44,7 +44,7 @@ _kill_ss = ss -K dst "[]" # Option: kill # Notes.: can be used to specify custom killing feature, by default depending on option kill-mode # Examples: banaction = ufw[kill='ss -K "( sport = :http || sport = :https )" dst "[]"'] - banaction = ufw[kill='cutter ""'] +# banaction = ufw[kill='cutter ""'] kill = <_kill_> [Init] From 3f9cf278534b7e8051590767c682a0114c5f6907 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 11 May 2021 13:47:48 +0200 Subject: [PATCH 086/139] filter.d/apache-fakegooglebot.conf: better, more precise regex and datepattern (closes possible weakness like #3013) --- config/filter.d/apache-fakegooglebot.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf index 729410ad..ee23656a 100644 --- a/config/filter.d/apache-fakegooglebot.conf +++ b/config/filter.d/apache-fakegooglebot.conf @@ -2,11 +2,11 @@ [Definition] -failregex = ^ .*Googlebot.*$ +failregex = ^\s* \S+ \S+(?: \S+)?\s+\S+ "[A-Z]+ /\S* [^"]*" \d+ \d+ \"[^"]*\" "[^"]*\bGooglebot/[^"]*" ignoreregex = -datepattern = ^[^\[]*\[({DATE}) +datepattern = ^[^\[]*(\[{DATE}\s*\]) {^LN-BEG} # DEV Notes: From 5e3de882af2b8f48e6de728c86b5483aad05a93c Mon Sep 17 00:00:00 2001 From: Ioannis Cherouvim <743305+cherouvim@users.noreply.github.com> Date: Wed, 12 May 2021 14:18:46 +0300 Subject: [PATCH 087/139] docs: Typo. (#3025) --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5f29161d..8ff407f0 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -271,7 +271,7 @@ effective ban duration (in seconds or time abbreviation format). time interval (in seconds or time abbreviation format) before the current time where failures will count towards a ban. .TP .B maxretry -number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP. +number of failures that have to occur in the last \fBfindtime\fR seconds to ban the IP. .TP .B backend backend to be used to detect changes in the logpath. From 2367ad115c1d20daaca886ebe4db81edf06df577 Mon Sep 17 00:00:00 2001 From: j-marz Date: Thu, 20 May 2021 09:15:45 +1000 Subject: [PATCH 088/139] fixed typo in comment --- config/filter.d/zoneminder.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf index 7da40968..b3c0be72 100644 --- a/config/filter.d/zoneminder.conf +++ b/config/filter.d/zoneminder.conf @@ -21,6 +21,6 @@ ignoreregex = # Notes: # Tested on Zoneminder 1.29 and 1.35.21 # -# Zoneminer versions > 1.3x use "ERR" and < 1.3x use "WAR" level logs, so i've kept both for compatibility reasons +# Zoneminder versions > 1.3x use "ERR" and < 1.3x use "WAR" level logs, so i've kept both for compatibility reasons # # Author: John Marzella From ec4e0dd65b1f2483a3e2413a61442eb544ae8c16 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 21 May 2021 13:00:24 +0200 Subject: [PATCH 089/139] padding with space, prefregex, regex review (simplifying, capture user name, consider possible space char in user name) --- config/filter.d/zoneminder.conf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf index b3c0be72..8e8ed432 100644 --- a/config/filter.d/zoneminder.conf +++ b/config/filter.d/zoneminder.conf @@ -6,15 +6,16 @@ before = apache-common.conf [Definition] # patterns: [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ -# [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ -# [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ +# [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ +# [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ # # Option: failregex # Notes.: regex to match the login failure and non-existent user error messages in the logfile. -failregex = ^%(_apache_error_client)s WAR \[Login denied for user "[^"]*"\] - ^%(_apache_error_client)s ERR \[Login denied for user "[^"]*"\] - ^%(_apache_error_client)s ERR \[Could not retrieve user \w* details\] +prefregex = ^%(_apache_error_client)s (?:ERR|WAR) \[(?:Login denied|Could not retrieve).*$ + +failregex = ^\[Login denied for user "[^"]*"\] + ^\[Could not retrieve user \S* ignoreregex = From 1627d4f573a1c763d00bee881a245e4c29c3f0b1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 May 2021 23:15:31 +0200 Subject: [PATCH 090/139] filter.d/sendmail-auth.conf: user not found, closes gh-3030 --- config/filter.d/sendmail-auth.conf | 1 + fail2ban/tests/files/logs/sendmail-auth | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 84fcbdda..4601a1fc 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -16,6 +16,7 @@ prefregex = ^%(__prefix_line)s.+$ failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ + ^AUTH failure \([^\)]+\):(?: [^:]+:)? user not found: [^,]*, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index 93bf0b14..baa0a33f 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -22,3 +22,8 @@ Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) # failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) + +# failJSON: { "time": "2005-05-24T01:58:40", "match": true , "host": "192.0.2.5", "desc": "user not found (gh-3030)" } +May 24 01:58:40 server sm-mta[65696]: 14NNwaRl065696: AUTH failure (DIGEST-MD5): user not found (-20) SASL(-13): user not found: unable to canonify user and get auxprops, user=scanner, relay=[192.0.2.5] +# failJSON: { "time": "2005-05-24T01:59:07", "match": true , "host": "192.0.2.6", "desc": "user not found (gh-3030)" } +May 24 01:59:07 server sm-mta[65815]: 14NNx65Q065815: AUTH failure (CRAM-MD5): user not found (-20) SASL(-13): user not found: user: scan@server.example.com property: userPassword not found in sasldb /usr/local/etc/sasldb2, user=scan, relay=[192.0.2.6] From 87f717e0e038afeb9b2ac98617fc3d907ddc49bc Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 17:45:02 +0200 Subject: [PATCH 091/139] filter.d/sendmail-reject.conf: fix reverse DNS for ... (gh-3012) --- config/filter.d/sendmail-reject.conf | 2 +- fail2ban/tests/files/logs/sendmail-reject | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index 966d880c..41035e5f 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -25,7 +25,7 @@ addr = (?:(?:IPv6:)?|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index ed3143a8..8debe7ca 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -72,6 +72,8 @@ Feb 13 01:16:50 batman sm-mta[25815]: s1D0GoSs025815: [217.193.142.180]: vrfy in # failJSON: { "time": "2005-02-22T14:02:44", "match": true , "host": "24.73.201.194" } Feb 22 14:02:44 batman sm-mta[4030]: s1MD2hsd004030: rrcs-24-73-201-194.se.biz.rr.com [24.73.201.194]: VRFY root [rejected] +# failJSON: { "time": "2005-02-22T15:20:27", "match": true , "host": "192.0.2.5", "desc": "Fix reverse DNS for ... (gh-3012)" } +Feb 22 15:20:27 localhost sm-mta[275631]: 13O9Ixhq275631: ruleset=check_rcpt, arg1=, relay=[192.0.2.5], reject=550 5.7.1 ... Fix reverse DNS for 192.0.2.5 # failJSON: { "match": false } Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026250: ... No such user here @@ -109,4 +111,4 @@ Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not i # failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 # failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } -Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 From ae3e9b9149c36e3334447e869667bcf171ff0a3a Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 19:21:27 +0200 Subject: [PATCH 092/139] filter.d/postfix.conf: extended to cover 2 new vectors: - RCPT from unknown, 504 5.5.2, need fully-qualified hostname, gh-2995 - 550 5.7.25 Client host rejected, gh-2996 review combining several regex to single one --- config/filter.d/postfix.conf | 10 +++------- fail2ban/tests/files/logs/postfix | 5 +++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index fb690fb0..17982f3e 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -12,16 +12,12 @@ before = common.conf _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])? _port = (?::\d+)? +_pref = (?:RCPT|EHLO|VRFY) prefregex = ^%(__prefix_line)s> .+$ mdpr-normal = (?:\w+: reject:|(?:improper command pipelining|too many errors) after \S+) -mdre-normal=^RCPT from [^[]*\[\]%(_port)s: 55[04] 5\.7\.1\s - ^RCPT from [^[]*\[\]%(_port)s: 45[04] 4\.7\.\d+ (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b) - ^RCPT from [^[]*\[\]%(_port)s: 450 4\.7\.\d+ (<[^>]*>)?: Helo command rejected: Host not found\b - ^EHLO from [^[]*\[\]%(_port)s: 504 5\.5\.\d+ (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b - ^(RCPT|VRFY) from [^[]*\[\]%(_port)s: 550 5\.1\.1\s - ^RCPT from [^[]*\[\]%(_port)s: 450 4\.1\.\d+ (<[^>]*>)?: Sender address rejected: Domain not found\b +mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|Client host rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b ^from [^[]*\[\]%(_port)s:? mdpr-auth = warning: @@ -31,7 +27,7 @@ mdre-auth2= ^[^[]*\[\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5 # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only: mdpr-rbl = %(mdpr-normal)s -mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b +mdre-rbl = ^%(_pref)s from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b # Mode "rbl" currently included in mode "normal" (within 1st rule) mdpr-more = %(mdpr-normal)s diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 6e2dc460..d46e6538 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -35,6 +35,11 @@ Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.exa # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= +# failJSON: { "time": "2005-04-06T13:05:01", "match": true , "host": "192.0.2.116", "desc": "RCPT from unknown, gh-2995" } +Apr 6 13:05:01 server postfix/smtpd[20589]: NOQUEUE: reject: RCPT from unknown[192.0.2.116]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; from= to= proto=ESMTP helo= +# failJSON: { "time": "2005-04-07T03:10:56", "match": true , "host": "192.0.2.246", "desc": "550 5.7.25 Client host rejected, gh-2996" } +Apr 7 03:10:56 server postfix/smtpd[7754]: NOQUEUE: reject: RCPT from unknown[192.0.2.246]: 550 5.7.25 Client host rejected: cannot find your hostname, [192.0.2.246]; from= to= proto=ESMTP helo=<[192.0.2.246]> + # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= From c5f1598a215b337ce5a93243fe17896edff780e6 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 19:48:24 +0200 Subject: [PATCH 093/139] filter.d/postfix.conf: extended to cover new vectors: - reject: BDAT/DATA from (gh-2927) - (since regex is more precise now) token selector changed to `[A-Z]{4}`, e. g. no matter what a command is supplied now (RCPT, EHLO, VRFY, DATA, BDAT or something else) - matches "Command rejected" and "Data command rejected" now --- config/filter.d/postfix.conf | 6 +++--- fail2ban/tests/files/logs/postfix | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 17982f3e..01d8cb0b 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -12,12 +12,12 @@ before = common.conf _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])? _port = (?::\d+)? -_pref = (?:RCPT|EHLO|VRFY) +_pref = [A-Z]{4} prefregex = ^%(__prefix_line)s> .+$ -mdpr-normal = (?:\w+: reject:|(?:improper command pipelining|too many errors) after \S+) -mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|Client host rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b +mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+) +mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b ^from [^[]*\[\]%(_port)s:? mdpr-auth = warning: diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index d46e6538..85b61ea6 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -35,6 +35,11 @@ Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.exa # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= +# failJSON: { "time": "2005-03-7T02:09:33", "match": true , "host": "192.0.2.151", "desc": "reject: DATA from, gh-2927" } +Mar 7 02:09:33 server postfix/smtpd[27246]: 1D8CC1CA0A7F: milter-reject: DATA from 66-220-155-151.mail-mail.facebook.com[192.0.2.151]: 550 5.7.1 Command rejected; from= to= proto=ESMTP helo=<192-0-2-151.mail-mail.example.com> +# failJSON: { "time": "2005-03-11T23:27:54", "match": true , "host": "192.0.2.109", "desc": "reject: BDAT from, gh-2927" } +Mar 11 23:27:54 server postfix-smo/submission/smtpd[22427]: 44JCRG5tYPzCqt2: reject: BDAT from signing-milter.example.com[192.0.2.109]: 550 5.5.3 : Data command rejected: Multi-recipient bounce; from=<> to= proto=ESMTP helo= + # failJSON: { "time": "2005-04-06T13:05:01", "match": true , "host": "192.0.2.116", "desc": "RCPT from unknown, gh-2995" } Apr 6 13:05:01 server postfix/smtpd[20589]: NOQUEUE: reject: RCPT from unknown[192.0.2.116]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; from= to= proto=ESMTP helo= # failJSON: { "time": "2005-04-07T03:10:56", "match": true , "host": "192.0.2.246", "desc": "550 5.7.25 Client host rejected, gh-2996" } From 8afea374943b6a2589bf3964f15af0aa17c6c369 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 20:09:57 +0200 Subject: [PATCH 094/139] filter.d/sendmail-auth.conf: covering several "authentication failure" messages, sendmail 8.16.1 (gh-2757) --- config/filter.d/sendmail-auth.conf | 3 +-- fail2ban/tests/files/logs/sendmail-auth | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 4601a1fc..de1f8e36 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -15,8 +15,7 @@ addr = (?:IPv6:|) prefregex = ^%(__prefix_line)s.+$ failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ - ^AUTH failure \([^\)]+\):(?: [^:]+:)? user not found: [^,]*, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ + ^AUTH failure \([^\)]+\):(?: [^:]+:)? (?:authentication failure|user not found): [^,]*, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index baa0a33f..f88cde86 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -27,3 +27,8 @@ Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): au May 24 01:58:40 server sm-mta[65696]: 14NNwaRl065696: AUTH failure (DIGEST-MD5): user not found (-20) SASL(-13): user not found: unable to canonify user and get auxprops, user=scanner, relay=[192.0.2.5] # failJSON: { "time": "2005-05-24T01:59:07", "match": true , "host": "192.0.2.6", "desc": "user not found (gh-3030)" } May 24 01:59:07 server sm-mta[65815]: 14NNx65Q065815: AUTH failure (CRAM-MD5): user not found (-20) SASL(-13): user not found: user: scan@server.example.com property: userPassword not found in sasldb /usr/local/etc/sasldb2, user=scan, relay=[192.0.2.6] + +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.7", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (login): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=test, relay=host.example.com [192.0.2.7] (may be forged) +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.8", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (PLAIN): authentication failure (-13) SASL(-13): authentication failure: Password verification failed, user=test, relay=host.example.com [192.0.2.8] From 6be1a5a0b1b74ff8c1f3cb07e540f24260a125bf Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 20:25:28 +0200 Subject: [PATCH 095/139] filter.d/dovecot.conf: fixed "Authentication failure" regex, matches "Password mismatch" in title case (gh-2880) --- config/filter.d/dovecot.conf | 4 ++-- fail2ban/tests/files/logs/dovecot | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 98d9f43b..7ec9a30c 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -14,8 +14,8 @@ prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_a failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ - ^pam\(\S+,(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$ - ^[a-z\-]{3,15}\(\S*,(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch) + ^pam\(\S+,(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$ + ^[a-z\-]{3,15}\(\S*,(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch) > mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index c1ed28f9..17a16b9f 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -34,6 +34,9 @@ Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSER # failJSON: { "time": "2005-01-29T05:32:50", "match": true , "host": "1.2.3.4" } Jan 29 05:32:50 mail dovecot: auth-worker(304): pam(username,1.2.3.4): pam_authenticate() failed: Authentication failure (password mismatch?) +# failJSON: { "time": "2005-01-29T18:55:55", "match": true , "host": "192.0.2.4", "desc": "Password mismatch (title case, gh-2880)" } +Jan 29 18:55:55 mail dovecot: auth-worker(12182): pam(user,192.0.2.4): pam_authenticate() failed: Authentication failure (Password mismatch?) + # failJSON: { "time": "2005-01-29T05:13:40", "match": true , "host": "1.2.3.4" } Jan 29 05:13:40 mail dovecot: auth-worker(31326): pam(username,1.2.3.4): unknown user From 8b984a0135be5ea284a2345cce0e3458dfb95d7b Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 20:47:56 +0200 Subject: [PATCH 096/139] filter.d\exim-common.conf: pid-prefix extended to match `mx1 exim[...]:` (gh-2553) --- config/filter.d/exim-common.conf | 2 +- fail2ban/tests/files/logs/exim | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index b3b25750..36644e94 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -12,7 +12,7 @@ after = exim-common.local host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)? host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s host_info = %(host_info_pre)s\[\]%(host_info_suf)s -pid = (?: \[\d+\])? +pid = (?: \[\d+\]| \w+ exim\[\d+\]:)? # DEV Notes: # From exim source code: ./src/receive.c:add_host_info_for_log diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 79437a90..e88f06ef 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -43,6 +43,9 @@ # failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" } 2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner) +# failJSON: { "time": "2019-10-22T03:39:17", "match": true , "host": "192.0.2.37", "desc": "pid-prefix in form of 'mx1 exim[...]:', gh-2553" } +2019-10-22 03:39:17 mx1 exim[29786]: dovecot_login authenticator failed for (User) [192.0.2.37]: 535 Incorrect authentication data (set_id=test@domain.com) + # failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" } 2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster) From 92f90038fa67d719c55c75f5eff8059e849fe747 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 21:12:34 +0200 Subject: [PATCH 097/139] filter.d/dovecot.conf: extended to match prefix like `conn unix:auth-worker (uid=143): auth-worker<13247>:` (authenticate from external service like exim), gh-2553 --- config/filter.d/dovecot.conf | 3 ++- fail2ban/tests/files/logs/dovecot | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 7ec9a30c..9c817720 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -8,9 +8,10 @@ before = common.conf [Definition] _auth_worker = (?:dovecot: )?auth(?:-worker)? +_auth_worker_info = (?:conn \w+:auth(?:-worker)? \(uid=\w+\): auth(?:-worker)?<\d+>: )? _daemon = (?:dovecot(?:-auth)?|auth) -prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?.+$ +prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s.+$ failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 17a16b9f..6bcf8c5b 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -55,6 +55,11 @@ Jun 11 13:57:17 main dovecot: auth: sql(admin@example.ru,192.168.144.226,<6rXunF #failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573" } Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,): Password mismatch (for LDAP bind) (SHA1 of given password: f638ff) +# failJSON: { "time": "2005-06-12T11:48:12", "match": true , "host": "192.0.2.6" } +Jun 12 11:48:12 auth-worker(80180): Info: conn unix:auth-worker (uid=143): auth-worker<13247>: sql(support,192.0.2.6): unknown user +# failJSON: { "time": "2005-06-12T23:06:05", "match": true , "host": "192.0.2.7" } +Jun 12 23:06:05 auth-worker(57065): Info: conn unix:auth-worker (uid=143): auth-worker<225622>: sql(user@domain.com,192.0.2.7,): Password mismatch + # failJSON: { "time": "2005-01-29T14:38:51", "match": true , "host": "192.0.2.6", "desc": "PAM Permission denied (gh-1897)" } Jan 29 14:38:51 example.com dovecot[24941]: auth-worker(30165): pam(user@example.com,192.0.2.6,): pam_authenticate() failed: Permission denied From c7a86b4616ef1bd43cf96874710b53b42d065684 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 May 2021 22:59:55 +0200 Subject: [PATCH 098/139] action.d/firewallcmd-ipset.conf: amend to #2620: - combines actions `firewallcmd-ipset` and `firewallcmd-ipset-native` (parameter `ipsettype=firewalld`); - IPv6-capability for firewalld ipset; - no internal timeout handling by default; - no permanent rules yet --- config/action.d/firewallcmd-ipset-native.conf | 77 ------------------- config/action.d/firewallcmd-ipset.conf | 43 +++++++++-- 2 files changed, 38 insertions(+), 82 deletions(-) delete mode 100644 config/action.d/firewallcmd-ipset-native.conf diff --git a/config/action.d/firewallcmd-ipset-native.conf b/config/action.d/firewallcmd-ipset-native.conf deleted file mode 100644 index 757d46ad..00000000 --- a/config/action.d/firewallcmd-ipset-native.conf +++ /dev/null @@ -1,77 +0,0 @@ -# Fail2Ban action file for firewall-cmd using native ipset implementation -# -# This requires: -# ipset (package: ipset) -# firewall-cmd (package: firewalld) -# -# This is for ipset protocol 6 (and hopefully later) (ipset v6.14). -# Use ipset -V to see the protocol and version. -# -# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. -# -# If you are running on an older kernel you make need to patch in external -# modules. - -[INCLUDES] - -before = firewallcmd-common.conf - -[Definition] - -actionstart = firewall-cmd --permanent --new-ipset= --type=hash:ip --option=timeout= - firewall-cmd --reload - firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j - -actionstop = firewall-cmd --direct --remove-rule filter 0 -m set --match-set src -j - firewall-cmd --permanent --delete-ipset= - firewall-cmd --reload - -actionban = firewall-cmd --ipset= --add-entry= - -actionunban = firewall-cmd --ipset= --remove-entry= - -[Init] - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: [ STRING ] -# -chain = INPUT_direct - -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 - -bantime = 86400 - -# Option: actiontype -# Notes.: defines additions to the blocking rule -# Values: leave empty to block all attempts from the host -# Default: Value of the multiport -actiontype = - -# Option: allports -# Notes.: default addition to block all ports -# Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -# for all protocols: banaction = firewallcmd-ipset[actiontype=""] -allports = -p - -# Option: multiport -# Notes.: addition to block access only to specific ports -# Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -multiport = -p -m multiport --dports - -ipmset = f2b- -familyopt = - -[Init?family=inet6] - -ipmset = f2b-6 -familyopt = family inet6 - - -# DEV NOTES: -# -# Author: Edgar Hoch and Daniel Black and Mihail Politaev -# firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index c89a0243..e1cb67e7 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,21 +18,46 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = /actionstart> firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j -actionflush = ipset flush +actionflush = /actionflush> actionstop = firewall-cmd --direct --remove-rule filter 0 -m set --match-set src -j - ipset destroy + /actionstop> -actionban = ipset add timeout -exist +actionban = /actionban> # actionprolong = %(actionban)s +actionunban = /actionunban> + +[ipstype_ipset] + +actionstart = ipset create hash:ip timeout + +actionflush = ipset flush + +actionstop = ipset destroy + +actionban = ipset add timeout -exist + actionunban = ipset del -exist +[ipstype_firewalld] + +actionstart = firewall-cmd --direct --new-ipset= --type=hash:ip --option=timeout= + +# TODO: there doesn't seem to be an explicit way to invoke the ipset flush function using firewall-cmd +actionflush = + +actionstop = firewall-cmd --direct --delete-ipset= + +actionban = firewall-cmd --ipset= --add-entry= + +actionunban = firewall-cmd --ipset= --remove-entry= + [Init] # Option: chain @@ -56,6 +81,12 @@ ipsettime = 0 # banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) +# Option: ipsettype +# Notes.: defines type of ipset used for match-set (firewalld or ipset) +# Values: firewalld or ipset +# Default: ipset +ipsettype = ipset + # Option: actiontype # Notes.: defines additions to the blocking rule # Values: leave empty to block all attempts from the host @@ -75,14 +106,16 @@ multiport = -p -m multiport --dports "$(echo '' | sed s/:/-/g)" ipmset = f2b- familyopt = +firewalld_familyopt = [Init?family=inet6] ipmset = f2b-6 familyopt = family inet6 +firewalld_familyopt = --option=family=inet6 # DEV NOTES: # -# Author: Edgar Hoch and Daniel Black +# Author: Edgar Hoch, Daniel Black, Sergey Brester and Mihail Politaev # firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness From bbfff1828061514e48395a5dbc5c1f9f81625e82 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 3 Jun 2021 12:02:08 +0200 Subject: [PATCH 099/139] action.d/ufw.conf: amend to #3018: parameter `kill-mode` extended with conntrack --- config/action.d/ufw.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index 088a5a9d..c9ff7f37 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -33,13 +33,14 @@ actionunban = if [ -n "" ] && ufw app info "" fi # Option: kill-mode -# Notes.: can be set to ss (may be extended later with other modes) to immediately drop all connections from banned IP, default empty (no kill) +# Notes.: can be set to ss or conntrack (may be extended later with other modes) to immediately drop all connections from banned IP, default empty (no kill) # Example: banaction = ufw[kill-mode=ss] kill-mode = # intern conditional parameter used to provide killing mode after ban: _kill_ = _kill_ss = ss -K dst "[]" +_kill_conntrack = conntrack -D -s "" # Option: kill # Notes.: can be used to specify custom killing feature, by default depending on option kill-mode From 43f2923fbd2507b6aa9c357ead261f1800555ec6 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 10 Jun 2021 15:06:54 +0200 Subject: [PATCH 100/139] filter.d/postfix.conf: matches rejects with "undeliverable address" (sender/recipient verification, gh-3039) additionally to "Unknown user"; both are configurable now via extended parameter and can be disabled using `exre-user=` supplied in filter parameters --- config/filter.d/postfix.conf | 5 ++++- fail2ban/tests/files/logs/postfix | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 72fa3c06..31c06605 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -16,8 +16,11 @@ _pref = [A-Z]{4} prefregex = ^%(__prefix_line)s> .+$ +# Extended RE for normal mode to match reject by unknown users or undeliverable address, can be set to empty to avoid this: +exre-user = |[Uu](?:ser unknown|ndeliverable address) + mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+) -mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b +mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match%(exre-user)s)\b ^from [^[]*\[\]%(_port)s:? mdpr-auth = warning: diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 580e0d03..c973377e 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -15,6 +15,9 @@ Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY fr # failJSON: { "time": "2005-08-13T15:45:46", "match": true , "host": "192.0.2.1" } Aug 13 15:45:46 server postfix/smtpd[13844]: 00ADB3C0899: reject: RCPT from example.com[192.0.2.1]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient table; from= to= proto=ESMTP helo= +# failJSON: { "time": "2005-05-19T00:00:30", "match": true , "host": "192.0.2.2", "desc": "undeliverable address (sender/recipient verification, gh-3039)" } +May 19 00:00:30 proxy2 postfix/smtpd[16123]: NOQUEUE: reject: RCPT from example.net[192.0.2.2]: 550 5.1.1 : Recipient address rejected: undeliverable address: verification failed; from= to= proto=ESMTP helo= + # failJSON: { "time": "2005-01-12T11:07:49", "match": true , "host": "181.21.131.88" } Jan 12 11:07:49 emf1pt2-2-35-70 postfix/smtpd[13767]: improper command pipelining after DATA from unknown[181.21.131.88]: From 579c6a94aff0bb680090b3be1aef91491cf0a061 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 10 Jun 2021 15:23:24 +0200 Subject: [PATCH 101/139] filter.d/postfix.conf: mode `ddos` (and `aggressive`) extended to consider abusive handling of clients hitting command limit (gh-3040) --- config/filter.d/postfix.conf | 2 +- fail2ban/tests/files/logs/postfix | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 31c06605..b374f472 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -38,7 +38,7 @@ mdre-more = %(mdre-normal)s # Includes some of the log messages described in # . -mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+) +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+|COMMAND (?:TIME|COUNT|LENGTH) LIMIT) mdre-ddos = ^from [^[]*\[\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index c973377e..d1e534e3 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -169,6 +169,12 @@ Dec 23 19:39:13 xxx postfix/postscreen[21057]: PREGREET 14 after 0.08 from [192. # failJSON: { "time": "2004-12-24T00:54:36", "match": true , "host": "192.0.2.3" } Dec 24 00:54:36 xxx postfix/postscreen[22515]: HANGUP after 16 from [192.0.2.3]:48119 in tests after SMTP handshake +# failJSON: { "time": "2005-06-08T23:14:28", "match": true , "host": "192.0.2.77", "desc": "abusive clients hitting command limit, see see http://www.postfix.org/POSTSCREEN_README.html (gh-3040)" } +Jun 8 23:14:28 proxy2 postfix/postscreen[473]: COMMAND TIME LIMIT from [192.0.2.77]:3608 after CONNECT +# failJSON: { "time": "2005-06-08T23:14:54", "match": true , "host": "192.0.2.26", "desc": "abusive clients hitting command limit (gh-3040)" } +Jun 8 23:14:54 proxy2 postfix/postscreen[473]: COMMAND COUNT LIMIT from [192.0.2.26]:15592 after RCPT + + # filterOptions: [{}, {"mode": "ddos"}, {"mode": "aggressive"}] # failJSON: { "match": false, "desc": "don't affect lawful data (sporadical connection aborts within DATA-phase, see gh-1813 for discussion)" } Feb 18 09:50:05 xxx postfix/smtpd[42]: lost connection after DATA from good-host.example.com[192.0.2.10] From 410a6ce5c80dd981c22752da034f2529b5eee844 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 21 Jun 2021 17:12:53 +0200 Subject: [PATCH 102/139] fixed possible RCE vulnerability, unset escape variable (default tilde) stops consider "~" char after new-line as composing escape sequence --- config/action.d/complain.conf | 2 +- config/action.d/dshield.conf | 2 +- config/action.d/mail-buffered.conf | 8 ++++---- config/action.d/mail-whois-lines.conf | 2 +- config/action.d/mail-whois.conf | 6 +++--- config/action.d/mail.conf | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 3a5f882c..4d73b058 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -102,7 +102,7 @@ logpath = /dev/null # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index c128bef3..3d5a7a53 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -179,7 +179,7 @@ tcpflags = # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf index 325f185b..79b84104 100644 --- a/config/action.d/mail-buffered.conf +++ b/config/action.d/mail-buffered.conf @@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Output will be buffered until lines are available.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -28,13 +28,13 @@ actionstop = if [ -f ]; then These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary from " rm fi printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -55,7 +55,7 @@ actionban = printf %%b "`date`: ( failures)\n" >> These hosts have been banned by Fail2Ban.\n `cat ` \nRegards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary" rm fi diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 3a3e56b2..d2818cb9 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -72,7 +72,7 @@ actionunban = # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Default name of the chain # diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index 7fea34c4..ab33b616 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -20,7 +20,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n Here is more information about :\n `%(_whois_command)s`\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf index 5d8c0e15..f4838ddc 100644 --- a/config/action.d/mail.conf +++ b/config/action.d/mail.conf @@ -16,7 +16,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the From 2ed414ed09b3bb4c478abc9366a1ff22024a33c9 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 21 Jun 2021 17:12:53 +0200 Subject: [PATCH 103/139] fixed possible RCE vulnerability, unset escape variable (default tilde) stops consider "~" char after new-line as composing escape sequence closes GHSA-m985-3f3v-cwmm for 0.9 --- config/action.d/complain.conf | 2 +- config/action.d/dshield.conf | 2 +- config/action.d/mail-buffered.conf | 8 ++++---- config/action.d/mail-whois-lines.conf | 12 +++++++++--- config/action.d/mail-whois.conf | 6 +++--- config/action.d/mail.conf | 6 +++--- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 9247803e..993a31e4 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -79,7 +79,7 @@ logpath = /dev/null # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index a0041986..ae7014fa 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -176,7 +176,7 @@ tcpflags = # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf index 914d4a5a..f530e958 100644 --- a/config/action.d/mail-buffered.conf +++ b/config/action.d/mail-buffered.conf @@ -14,7 +14,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Output will be buffered until lines are available.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -25,13 +25,13 @@ actionstop = if [ -f ]; then These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary from `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary from `uname -n`" rm fi printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -52,7 +52,7 @@ actionban = printf %%b "`date`: ( failures)\n" >> These hosts have been banned by Fail2Ban.\n `cat ` \nRegards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary" rm fi diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 6e39c605..39f78957 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -17,7 +17,7 @@ before = mail-whois-common.conf actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban" | "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -26,7 +26,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban" | "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -48,7 +48,7 @@ actionban = printf %%b "Hi,\n Lines containing IP: in \n `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban" | "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -60,6 +60,12 @@ actionunban = [Init] +# Option: mailcmd +# Notes.: Your system mail command. Is passed 2 args: subject and recipient +# Values: CMD +# +mailcmd = mail -E 'set escape' -s + # Default name of the chain # name = default diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index 018c327d..2c346bfe 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -17,7 +17,7 @@ before = mail-whois-common.conf actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -26,7 +26,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -46,7 +46,7 @@ actionban = printf %%b "Hi,\n Here is more information about :\n `%(_whois_command)s`\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf index 7bf51a1d..9755f097 100644 --- a/config/action.d/mail.conf +++ b/config/action.d/mail.conf @@ -13,7 +13,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -22,7 +22,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -40,7 +40,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the From 39fe0bdce612b2a3cb649866da8247d51f5731ab Mon Sep 17 00:00:00 2001 From: Alex Porto dos Santos <31667409+alexporto2200@users.noreply.github.com> Date: Tue, 17 Aug 2021 10:39:04 -0300 Subject: [PATCH 104/139] Update Readme Add python3-setuptools or python-setuptools for dependencies. On some distributions this doesn't come by default, it would be nice to let users know about this in the documentation. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fac3414d..060b250f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ this case, you should use that instead.** Required: - [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org) +- python3-setuptools or python-setuptools for run installer. Optional: - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require: From ade79635b27c408b811ffb3dd9a806a0963771c5 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 17 Aug 2021 16:41:20 +0200 Subject: [PATCH 105/139] distutils/setuptools only required if installing from source --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 060b250f..ba5c149e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ this case, you should use that instead.** Required: - [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org) -- python3-setuptools or python-setuptools for run installer. +- python-setuptools, python-distutils or python3-setuptools for installation from source Optional: - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require: From 2f99d5accb5b349817c4f8f3dc73a0162b05cab3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Sep 2021 18:22:31 +0200 Subject: [PATCH 106/139] test coverage for unhandled exception in run of several filter (gh-3097) --- fail2ban/server/filter.py | 2 +- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 2 +- fail2ban/server/filtersystemd.py | 2 +- fail2ban/tests/filtertestcase.py | 18 ++++++++++++++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index f514b337..e9000447 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -722,7 +722,7 @@ class Filter(JailThread): # incr common error counter: self.commonError() - def commonError(self): + def commonError(self, reason="common", exc=None): # incr error counter, stop processing (going idle) after 100th error : self._errors += 1 # sleep a little bit (to get around time-related errors): diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 7ee00540..196955e5 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -122,7 +122,7 @@ class FilterPoll(FileFilter): logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) return True diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index d62348a2..b9936df5 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -363,7 +363,7 @@ class FilterPyinotify(FileFilter): logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 925109d1..d70f9259 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -334,7 +334,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 3cc17fb1..5166bc43 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -979,6 +979,10 @@ class CommonMonitorTestCase(unittest.TestCase): super(CommonMonitorTestCase, self).setUp() self._failTotal = 0 + def tearDown(self): + super(CommonMonitorTestCase, self).tearDown() + self.assertFalse(hasattr(self, "_unexpectedError")) + def waitFailTotal(self, count, delay=1): """Wait up to `delay` sec to assure that expected failure `count` reached """ @@ -1004,6 +1008,16 @@ class CommonMonitorTestCase(unittest.TestCase): last_ticks = self.filter.ticks return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay)) + def commonFltError(self, reason="common", exc=None): + """ Mock-up for default common error handler to find catched unhandled exceptions + could occur in filters + """ + self._commonFltError(reason, exc) + if reason == "unhandled": + DefLogSys.critical("Caught unhandled exception in main cycle of %r : %r", self.filter, exc, exc_info=True) + self._unexpectedError = True + # self.assertNotEqual(reason, "unhandled") + def get_monitor_failures_testcase(Filter_): """Generator of TestCase's for different filters/backends @@ -1026,6 +1040,8 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1324,6 +1340,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", From 1e4a14fb25d88e32f3ca9c06fb1d6b8d3b4813ab Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Sep 2021 19:16:49 +0200 Subject: [PATCH 107/139] pyinotify: fixes sporadic runtime error "dictionary changed size during iteration" (if something outside changes the pending dict during _checkPending evaluation) - simply deserialize to a list for iteration, without any lock, because unneeded here due to small and mostly empty dictionary (logrotate, etc), not to mention that pending check is normally called once per minute; don't call process file inside of server thread calling of addLogPath (always retard it as pending event); ensure to wake-up as soon as possible to process pending events (e. g. if file gets added). --- fail2ban/server/filterpyinotify.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index b9936df5..5f449cad 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -165,7 +165,7 @@ class FilterPyinotify(FileFilter): return found = {} minTime = 60 - for path, (retardTM, isDir) in self.__pending.iteritems(): + for path, (retardTM, isDir) in list(self.__pending.items()): if ntm - self.__pendingChkTime < retardTM: if minTime > retardTM: minTime = retardTM continue @@ -268,15 +268,13 @@ class FilterPyinotify(FileFilter): def _addLogPath(self, path): self._addFileWatcher(path) - # initial scan: + # notify (wake up if in waiting): if self.active: - # we can execute it right now: - self._process_file(path) - else: - # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): - self._addPending(path, ('INITIAL', path), isDir=None) + self.__pendingMinTime = 0 + # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): + self._addPending(path, ('INITIAL', path), isDir=None) - ## + ## # Delete a log path # # @param path the log file to delete @@ -341,12 +339,17 @@ class FilterPyinotify(FileFilter): self.__notifier.process_events() # wait for events / timeout: - notify_maxtout = self.__notify_maxtout def __check_events(): - return not self.active or self.__notifier.check_events(timeout=notify_maxtout) - if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)): + return ( + not self.active + or bool(self.__notifier.check_events(timeout=self.__notify_maxtout)) + or (self.__pendingMinTime and self.__pending) + ) + wres = Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)) + if wres: if not self.active: break - self.__notifier.read_events() + if not isinstance(wres, dict): + self.__notifier.read_events() self.ticks += 1 From e323c148e13a141b22ea047b04a83a4cf248a7ac Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Sep 2021 19:42:08 +0200 Subject: [PATCH 108/139] backend systemd: fixes error "local variable 'line' referenced before assignment", introduced in 55d7d9e214f72bbe4f39a2d17aa004d80bfc7299; don't update database too often (every 10 ticks or ~ 10 seconds in production); closes gh-3097 --- fail2ban/server/filtersystemd.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index d70f9259..88f8c292 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -61,6 +61,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # Initialise systemd-journal connection self.__journal = journal.Reader(**jrnlargs) self.__matches = [] + self.__nextUpdateTM = 0 self.setDatePattern(None) logSys.debug("Created FilterSystemd") @@ -285,6 +286,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover except OSError: pass # Reading failure, so safe to ignore + line = None while self.active: # wait for records (or for timeout in sleeptime seconds): try: @@ -326,8 +328,15 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if self.ticks % 10 == 0: self.performSvc() # update position in log (time and iso string): - if self.jail.database is not None: + if (line and self.jail.database and ( + self.ticks % 10 == 0 + or MyTime.time() >= self.__nextUpdateTM + or not self.active + ) + ): self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 + line = None except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break From ba282b794c97279906c26fdcae93180621bc3067 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Sep 2021 19:56:02 +0200 Subject: [PATCH 109/139] pyinotify: amend to 1e4a14fb25d88e32f3ca9c06fb1d6b8d3b4813ab: one fix more for sporadic runtime error "dictionary changed size during iteration" (watched files) --- fail2ban/server/filterpyinotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 5f449cad..16b6cfd5 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -188,7 +188,7 @@ class FilterPyinotify(FileFilter): self._refreshWatcher(path, isDir=isDir) if isDir: # check all files belong to this dir: - for logpath in self.__watchFiles: + for logpath in list(self.__watchFiles): if logpath.startswith(path + pathsep): # if still no file - add to pending, otherwise refresh and process: if not os.path.isfile(logpath): @@ -285,7 +285,7 @@ class FilterPyinotify(FileFilter): logSys.error("Failed to remove watch on path: %s", path) path_dir = dirname(path) - for k in self.__watchFiles: + for k in list(self.__watchFiles): if k.startswith(path_dir + pathsep): path_dir = None break From d709ec8179e9a589bc2d979b5f33192b1ce023f5 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Sep 2021 20:00:41 +0200 Subject: [PATCH 110/139] GH actions: use newest python version for 3.10 (3.10.0-rc.2) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ea0e7eb..231cca1e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-beta.1', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-rc.2', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 1414a44b8ee166e63b28c5f4d800b821fa9ea62c Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 19 Sep 2021 18:24:36 +0200 Subject: [PATCH 111/139] Update main.yml CI: try to install dependencies via apt, add build test --- .github/workflows/main.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 231cca1e..c60ab05c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,12 +44,15 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + #python -m pip install --upgrade pip if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then - pip install 2to3 + #pip install 2to3 + sudo apt-get -y install 2to3 fi - pip install systemd-python || echo 'systemd not available' - pip install pyinotify || echo 'inotify not available' + #pip install systemd-python || echo 'systemd not available' + #pip install pyinotify || echo 'inotify not available' + sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' + sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' - name: Before scripts run: | @@ -61,6 +64,9 @@ jobs: - name: Test suite run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi - + + - name: Build + run: python setup.py build + #- name: Test initd scripts # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 7f22c4873aed3b5ffce0953f079f3c1977297c9a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 19 Sep 2021 18:36:02 +0200 Subject: [PATCH 112/139] remove 2to3 in setup (should be called outside before setup) --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 2e2a77fa..f4c2550f 100755 --- a/setup.py +++ b/setup.py @@ -39,14 +39,6 @@ from distutils.command.build_scripts import build_scripts if setuptools is None: from distutils.command.install import install from distutils.command.install_scripts import install_scripts -try: - # python 3.x - from distutils.command.build_py import build_py_2to3 - from distutils.command.build_scripts import build_scripts_2to3 - _2to3 = True -except ImportError: - # python 2.x - _2to3 = False import os from os.path import isfile, join, isdir, realpath From 5ac303df8a171f748330d4c645ccbf1c2c7f3497 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:49:18 +0200 Subject: [PATCH 113/139] fix gh-3098: build fails with error in fail2ban setup command: use_2to3 is invalid (setuptools 58+) --- setup.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/setup.py b/setup.py index f4c2550f..98413273 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ import warnings from glob import glob from fail2ban.setup import updatePyExec - +from fail2ban.version import version source_dir = os.path.realpath(os.path.dirname( # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): @@ -112,22 +112,12 @@ class install_scripts_f2b(install_scripts): # Wrapper to specify fail2ban own options: class install_command_f2b(install): user_options = install.user_options + [ - ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): - self.disable_2to3 = None self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): - global _2to3 - ## in the test cases 2to3 should be already done (fail2ban-2to3): - if self.disable_2to3: - _2to3 = False - if _2to3: - cmdclass = self.distribution.cmdclass - cmdclass['build_py'] = build_py_2to3 - cmdclass['build_scripts'] = build_scripts_2to3 if self.without_tests: self.distribution.scripts.remove('bin/fail2ban-testcases') @@ -178,7 +168,6 @@ commands.''' if setuptools: setup_extra = { 'test_suite': "fail2ban.tests.utils.gatherTests", - 'use_2to3': True, } else: setup_extra = {} @@ -202,9 +191,6 @@ if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswit ('/usr/share/doc/fail2ban', doc_files) ) -# Get version number, avoiding importing fail2ban. -# This is due to tests not functioning for python3 as 2to3 takes place later -exec(open(join("fail2ban", "version.py")).read()) setup( name = "fail2ban", From d6b884f3b72b8a42b21da863836569ef6836c2ea Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:52:34 +0200 Subject: [PATCH 114/139] amend to fix gh-3098: no option `--disable-2to3` anymore --- fail2ban/tests/misctestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 458e9a23..559a89f3 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -111,7 +111,7 @@ class SetupTest(unittest.TestCase): supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: # try dry-run: - os.system("%s %s --dry-run install --disable-2to3 --root=%s%s" + os.system("%s %s --dry-run install --root=%s%s" % (sys.executable, self.setup , tmp, supdbgout)) # check nothing was created: self.assertTrue(not os.listdir(tmp)) @@ -127,7 +127,7 @@ class SetupTest(unittest.TestCase): # suppress stdout (and stderr) if not heavydebug supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: - self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s" + self.assertEqual(os.system("%s %s install --root=%s%s" % (sys.executable, self.setup, tmp, supdbgout)), 0) def strippath(l): From a98cc08b31b41e9d344e26c6eb0d63d54b5f1242 Mon Sep 17 00:00:00 2001 From: Daniel Brooks Date: Sat, 10 Jul 2021 21:34:46 -0700 Subject: [PATCH 115/139] Updated the warning messages created when fail2ban sees unexpected timestamps to improve their grammar and to remove jargon. Partially fixes #2822 --- fail2ban/server/filter.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e9000447..f9ea9025 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -662,9 +662,10 @@ class Filter(JailThread): if (date is None or date < MyTime.time() - 60 or date > MyTime.time() + 60): # log time zone issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", - ("Simulate NOW in operation since found time has too large deviation %s ~ %s +/- %s", - date, MyTime.time(), 60), - ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + ("Found at least one log entry with a timestamp more" + + " than a minute from the current time. This is rarely a problem."), + ("Please check this jail and associated logs as" + + " there is potentially a timezone or latency problem: %s", line)) # simulate now as date: date = MyTime.time() @@ -674,9 +675,11 @@ class Filter(JailThread): if date is not None and date < MyTime.time() - self.getFindTime(): # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", - ("Ignore line since time %s < %s - %s", - date, MyTime.time(), self.getFindTime()), - ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + ("Ignoring all log entries older than %s; these are probably" + + " messages generated while fail2ban was not running.", + self.getFindTime()), + ("Please check this jail and associated logs as" + + " there is potentially a timezone or latency problem: %s", line)) # ignore - too old (obsolete) entry: return [] From 320a3dcdd5a8b3019df3dfb21022dd4d80851f38 Mon Sep 17 00:00:00 2001 From: Daniel Brooks Date: Sat, 10 Jul 2021 21:48:32 -0700 Subject: [PATCH 116/139] remove old warnings from filtertestcase.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assertLogged only checks that at least one listed message is found, so it isn’t necessary to repeat them in the test. --- fail2ban/tests/filtertestcase.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 5166bc43..9e81006b 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -457,8 +457,6 @@ class IgnoreIP(LogCaptureTestCase): for i in (1,2,3): self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 self.assertLogged( - "Simulate NOW in operation since found time has too large deviation", - "Please check jail has possibly a timezone issue.", "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", "Total # of detected failures: 3.", wait=True) # From 1929e7a76b03becbece228edd8c65204a7e83371 Mon Sep 17 00:00:00 2001 From: Daniel Brooks Date: Mon, 12 Jul 2021 09:21:00 -0700 Subject: [PATCH 117/139] include more specific information in the warning --- fail2ban/server/filter.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index f9ea9025..e327228e 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -659,14 +659,19 @@ class Filter(JailThread): # if in operation (modifications have been really found): if self.inOperation: # if weird date - we'd simulate now for timeing issue (too large deviation from now): - if (date is None or date < MyTime.time() - 60 or date > MyTime.time() + 60): + delta = date - MyTime.time() + if (date is None or abs(delta) > 60): + latency = "This could indicate a latency problem." + timezone = "This looks like a timezone problem." # log time zone issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", - ("Found at least one log entry with a timestamp more" + - " than a minute from the current time. This is rarely a problem."), - ("Please check this jail and associated logs as" + - " there is potentially a timezone or latency problem: %s", - line)) + ("Found a log entry with a timestamp %ss %s the current time. %s", + abs(delta), + "before" if delta <= 0 else "after", + latency if -15*60 <= delta <= 0 else timezone), + ("Please check this jail and associated logs as there" + + " is potentially a timezone or latency problem: %s", + line)) # simulate now as date: date = MyTime.time() self.__lastDate = date From d7afcde2e10fa61bb4adb6604862074461e19f16 Mon Sep 17 00:00:00 2001 From: Daniel Brooks Date: Mon, 12 Jul 2021 17:09:58 -0700 Subject: [PATCH 118/139] add a warning message for dates in the future and a test that checks which message was output for which time deltas. --- fail2ban/server/filter.py | 11 ++++++++++- fail2ban/tests/filtertestcase.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e327228e..16e7d2a6 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -662,13 +662,22 @@ class Filter(JailThread): delta = date - MyTime.time() if (date is None or abs(delta) > 60): latency = "This could indicate a latency problem." + synchronization = "This could be a clock synchronization" + \ + " problem; are you sure that NTP is set up?" timezone = "This looks like a timezone problem." + msg = "" + if -15*60 <= delta <= 0: + msg = latency + elif 0 < delta <= 15*60: + msg = synchronization + else: + msg = timezone # log time zone issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", ("Found a log entry with a timestamp %ss %s the current time. %s", abs(delta), "before" if delta <= 0 else "after", - latency if -15*60 <= delta <= 0 else timezone), + msg), ("Please check this jail and associated logs as there" + " is potentially a timezone or latency problem: %s", line)) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 9e81006b..6b0218f0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -479,6 +479,36 @@ class IgnoreIP(LogCaptureTestCase): finally: tearDownMyTime() + def testWrongTimeDelta(self): + try: + self.filter.addFailRegex('fail from $') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = True; # real processing (all messages are new) + expectable_messages = ["timezone problem.", "clock synchronization problem;", "latency problem."] + cases = [(-90*60, 0), + (-10*60, 1), + (-30, None), + (30, None), + (10*60, 2), + (90*60, 0)] + for idx, (delta, expect) in enumerate(cases): + MyTime.setTime(1572138000+delta) + setattr(self.filter, "_next_simByTimeWarn", -1) + self.pruneLog('[phase {phase}] log entries offset by {delta}s'.format(phase=idx+1, delta=delta)) + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); + for (idx, msg) in enumerate(expectable_messages): + if idx == expect: + self.assertLogged(expectable_messages[idx]) + else: + self.assertNotLogged(expectable_messages[idx]) + self.assertLogged( + "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", + "Total # of detected failures: 3.", wait=True) + finally: + tearDownMyTime() + def testAddAttempt(self): self.filter.setMaxRetry(3) for i in xrange(1, 1+3): From ec043cd2029fe52d56cf526fac31e79b79d73467 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 21:58:42 +0200 Subject: [PATCH 119/139] simplifying logic and shortening messages (delta in minutes; removed clock synchronization, because it is rarely an issue on fail2ban side, e. g. for remote logs only, etc) --- fail2ban/server/filter.py | 41 +++++++++--------------- fail2ban/tests/filtertestcase.py | 53 ++++++++++++++------------------ 2 files changed, 38 insertions(+), 56 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 16e7d2a6..08bc2e59 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -655,46 +655,35 @@ class Filter(JailThread): tupleLine = ("", self.__lastTimeText, line) date = self.__lastDate - if self.checkFindTime: + if self.checkFindTime and date is not None: # if in operation (modifications have been really found): if self.inOperation: # if weird date - we'd simulate now for timeing issue (too large deviation from now): - delta = date - MyTime.time() - if (date is None or abs(delta) > 60): - latency = "This could indicate a latency problem." - synchronization = "This could be a clock synchronization" + \ - " problem; are you sure that NTP is set up?" - timezone = "This looks like a timezone problem." - msg = "" - if -15*60 <= delta <= 0: - msg = latency - elif 0 < delta <= 15*60: - msg = synchronization - else: - msg = timezone - # log time zone issue as warning once per day: + delta = int(date - MyTime.time()) + if abs(delta) > 60: + delta /= 60 + # log timing issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", - ("Found a log entry with a timestamp %ss %s the current time. %s", - abs(delta), - "before" if delta <= 0 else "after", - msg), - ("Please check this jail and associated logs as there" + - " is potentially a timezone or latency problem: %s", + ("Detected a log entry %sm %s the current time in operation mode. " + "This looks like a %s problem. Treating such entries as if they just happened.", + abs(delta), "before" if delta < 0 else "after", + "latency" if -55 <= delta < 0 else "timezone" + ), + ("Please check a jail for a timing issue. Line with odd timestamp: %s", line)) # simulate now as date: date = MyTime.time() self.__lastDate = date else: # in initialization (restore) phase, if too old - ignore: - if date is not None and date < MyTime.time() - self.getFindTime(): + if date < MyTime.time() - self.getFindTime(): # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", - ("Ignoring all log entries older than %s; these are probably" + + ("Ignoring all log entries older than %ss; these are probably" + " messages generated while fail2ban was not running.", self.getFindTime()), - ("Please check this jail and associated logs as" + - " there is potentially a timezone or latency problem: %s", - line)) + ("Please check a jail for a timing issue. Line with odd timestamp: %s", + line)) # ignore - too old (obsolete) entry: return [] diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 6b0218f0..8e151f77 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -444,11 +444,11 @@ class IgnoreIP(LogCaptureTestCase): def testTimeJump_InOperation(self): self._testTimeJump(inOperation=True) - def testWrongTimeZone(self): + def testWrongTimeOrTZ(self): try: self.filter.addFailRegex('fail from $') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') - self.filter.setMaxRetry(5); # don't ban here + self.filter.setMaxRetry(50); # don't ban here self.filter.inOperation = True; # real processing (all messages are new) # current time is 1h later than log-entries: MyTime.setTime(1572138000+3600) @@ -476,36 +476,29 @@ class IgnoreIP(LogCaptureTestCase): "Match without a timestamp:", "192.0.2.17:1", "192.0.2.17:2", "192.0.2.17:3", "Total # of detected failures: 9.", all=True, wait=True) - finally: - tearDownMyTime() - - def testWrongTimeDelta(self): - try: - self.filter.addFailRegex('fail from $') - self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') - self.filter.setMaxRetry(5); # don't ban here - self.filter.inOperation = True; # real processing (all messages are new) - expectable_messages = ["timezone problem.", "clock synchronization problem;", "latency problem."] - cases = [(-90*60, 0), - (-10*60, 1), - (-30, None), - (30, None), - (10*60, 2), - (90*60, 0)] - for idx, (delta, expect) in enumerate(cases): + # + phase = 3 + for delta, expect in ( + (-90*60, "timezone"), #90 minutes after + (-60*60, "timezone"), #60 minutes after + (-10*60, "timezone"), #10 minutes after + (-59, None), #59 seconds after + (59, None), #59 seconds before + (61, "latency"), #>1 minute before + (55*60, "latency"), #55 minutes before + (90*60, "timezone") #90 minutes before + ): + phase += 1 MyTime.setTime(1572138000+delta) setattr(self.filter, "_next_simByTimeWarn", -1) - self.pruneLog('[phase {phase}] log entries offset by {delta}s'.format(phase=idx+1, delta=delta)) - for i in (1,2,3): - self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); - for (idx, msg) in enumerate(expectable_messages): - if idx == expect: - self.assertLogged(expectable_messages[idx]) - else: - self.assertNotLogged(expectable_messages[idx]) - self.assertLogged( - "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", - "Total # of detected failures: 3.", wait=True) + self.pruneLog('[phase {phase}] log entries offset by {delta}s'.format(phase=phase, delta=delta)) + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); + self.assertLogged("Found 192.0.2.15", wait=True) + if expect: + self.assertLogged(("timezone problem", "latency problem")[int(expect == "latency")], all=True) + self.assertNotLogged(("timezone problem", "latency problem")[int(expect != "latency")]) + else: + self.assertNotLogged("timezone problem", "latency problem", all=True) finally: tearDownMyTime() From 621d8cae17f16cfea43a394149217963034d344b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Sep 2021 02:20:22 +0200 Subject: [PATCH 120/139] restore backwards compatibility for date None --- fail2ban/server/filter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 08bc2e59..f49fe198 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -618,6 +618,7 @@ class Filter(JailThread): noDate = False if date: tupleLine = line + line = "".join(line) self.__lastTimeText = tupleLine[1] self.__lastDate = date else: @@ -654,6 +655,8 @@ class Filter(JailThread): if self.__lastDate and self.__lastDate > MyTime.time() - 60: tupleLine = ("", self.__lastTimeText, line) date = self.__lastDate + elif self.checkFindTime and self.inOperation: + date = MyTime.time() if self.checkFindTime and date is not None: # if in operation (modifications have been really found): From 17eed32e03e86f6b42a0f0ad227345713b7c7cd7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 21 Sep 2021 16:00:37 +0200 Subject: [PATCH 121/139] Update filtertestcase.py --- fail2ban/tests/filtertestcase.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 8e151f77..be874697 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -457,13 +457,18 @@ class IgnoreIP(LogCaptureTestCase): for i in (1,2,3): self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 self.assertLogged( + "Detected a log entry 60m before the current time in operation mode. This looks like a timezone problem.", + "Please check a jail for a timing issue.", "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", - "Total # of detected failures: 3.", wait=True) + "Total # of detected failures: 3.", all=True, wait=True) # + setattr(self.filter, "_next_simByTimeWarn", -1) self.pruneLog("[phase 2] wrong TZ given in log") for i in (1,2,3): self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 self.assertLogged( + "Detected a log entry 120m after the current time in operation mode. This looks like a timezone problem.", + "Please check a jail for a timing issue.", "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", "Total # of detected failures: 6.", all=True, wait=True) self.assertNotLogged("Found a match but no valid date/time found") @@ -496,7 +501,7 @@ class IgnoreIP(LogCaptureTestCase): self.assertLogged("Found 192.0.2.15", wait=True) if expect: self.assertLogged(("timezone problem", "latency problem")[int(expect == "latency")], all=True) - self.assertNotLogged(("timezone problem", "latency problem")[int(expect != "latency")]) + self.assertNotLogged(("timezone problem", "latency problem")[int(expect != "latency")], all=True) else: self.assertNotLogged("timezone problem", "latency problem", all=True) finally: From d086317cc83ec7aa6e6772dc91525bce33ed7200 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 21 Sep 2021 16:05:53 +0200 Subject: [PATCH 122/139] Update filter.py --- fail2ban/server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index f49fe198..a13b1fb5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -664,7 +664,7 @@ class Filter(JailThread): # if weird date - we'd simulate now for timeing issue (too large deviation from now): delta = int(date - MyTime.time()) if abs(delta) > 60: - delta /= 60 + delta //= 60 # log timing issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", ("Detected a log entry %sm %s the current time in operation mode. " From f8f59dd31a46f6ad35b38df1f29a8f81c7e88960 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 1 Oct 2021 14:58:25 +0200 Subject: [PATCH 123/139] added test cases covering different messages adjusted to new log-format (gh-3116) --- fail2ban/tests/files/logs/lighttpd-auth | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/lighttpd-auth b/fail2ban/tests/files/logs/lighttpd-auth index 184dba33..c8a922b5 100644 --- a/fail2ban/tests/files/logs/lighttpd-auth +++ b/fail2ban/tests/files/logs/lighttpd-auth @@ -1,4 +1,3 @@ -#authentification failure (mod_auth) # failJSON: { "time": "2011-12-25T17:09:20", "match": true , "host": "4.4.4.4" } 2011-12-25 17:09:20: (http_auth.c.875) password doesn't match for /gitweb/ username: francois, IP: 4.4.4.4 # failJSON: { "time": "2012-09-26T10:24:35", "match": true , "host": "4.4.4.4" } @@ -7,3 +6,9 @@ 2013-08-25 00:24:55: (http_auth.c.877) get_password failed, IP: 4.4.4.4 # failJSON: { "time": "2018-01-16T14:10:32", "match": true , "host": "192.0.2.1", "desc": "http_auth -> mod_auth, gh-2018" } 2018-01-16 14:10:32: (mod_auth.c.525) password doesn't match for /test-url username: test, IP: 192.0.2.1 +# failJSON: { "time": "2021-09-30T16:05:33", "match": true , "host": "192.0.2.2", "user":"test", "desc": "gh-3116" } +2021-09-30 16:05:33: mod_auth.c.828) password doesn't match for /secure/ username: test IP: 192.0.2.2 +# failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.3", "user":"tester", "desc": "gh-3116" } +2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed for tester : wrong password, IP: 192.0.2.3 +# failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.4", "desc": "gh-3116" } +2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed: uri mismatch (/uri1 != /uri2), IP: 192.0.2.4 From ba839af8ad594439002f468b71b4c6caafb5de03 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 1 Oct 2021 15:03:24 +0200 Subject: [PATCH 124/139] filter.d/lighttpd-auth.conf: adjusted to the current source code + avoiding catch-all's, etc (gh-3116) --- config/filter.d/lighttpd-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf index a68f4f4d..dcf19d3e 100644 --- a/config/filter.d/lighttpd-auth.conf +++ b/config/filter.d/lighttpd-auth.conf @@ -3,7 +3,7 @@ [Definition] -failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: \s*$ +failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ ignoreregex = From 1e8ce20b325e1a60c92025e0512fe0a547552293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Sun, 17 Oct 2021 19:53:10 +0200 Subject: [PATCH 125/139] Display ChangeLog as a Markdown document --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..0cbdbf83 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +ChangeLog linguist-language=Markdown From d91d949e95d5fbf70ca8679319dac7c8c2ba7a61 Mon Sep 17 00:00:00 2001 From: Mike Gilbert Date: Tue, 8 Jun 2021 17:48:12 -0400 Subject: [PATCH 126/139] tests: improve detection of readable systemd journal Look for system.journal in journal sub-directory. Add -readable to the find command. Bug: https://bugs.gentoo.org/794931 --- fail2ban/tests/filtertestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index be874697..2b3dd995 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1396,7 +1396,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover # check one at at time until the first hit for systemd_var in 'system-runtime-logs', 'system-state-logs': tmp = Utils.executeCmd( - 'find "$(systemd-path %s)" -name system.journal' % systemd_var, + 'find "$(systemd-path %s)/journal" -name system.journal -readable' % systemd_var, timeout=10, shell=True, output=True ) self.assertTrue(tmp) From 1cd3118da37b2281f841adf298fa20eff81b5017 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 23 Oct 2021 17:05:30 +0200 Subject: [PATCH 127/139] Fix typo found by lintian (Debian static analyzer) --- ChangeLog | 2 +- fail2ban/protocol.py | 2 +- man/fail2ban-client.1 | 2 +- man/jail.conf.5 | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 175d087a..a022f17c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -222,7 +222,7 @@ Yes, Hrrrm... ### New Features * new replacement tags for failregex to match subnets in form of IP-addresses with CIDR mask (gh-2559): - `` - helper regex to match CIDR (simple integer form of net-mask); - - `` - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional); + - `` - regex to match sub-net addresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional); * grouped tags (``, ``, ``) recognize IP addresses enclosed in square brackets * new failregex-flag tag `` for failregex, signaled that the access to service was gained (ATM used similar to tag ``, but it does not add the log-line to matches, gh-2279) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 0a4c84ed..58102b55 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -134,7 +134,7 @@ protocol = [ ["get ignoreregex", "gets the list of regular expressions which matches patterns to ignore for "], ["get findtime", "gets the time for which the filter will look back for failures for "], ["get bantime", "gets the time a host is banned for "], -["get datepattern", "gets the patern used to match date/times for "], +["get datepattern", "gets the pattern used to match date/times for "], ["get usedns", "gets the usedns setting for "], ["get banip [|--with-time]", "gets the list of of banned IP addresses for . Optionally the separator character ('', default is space) or the option '--with-time' (printing the times of ban) may be specified. The IPs are ordered by end of ban."], ["get maxretry", "gets the number of failures allowed for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 81473daa..2da467ec 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -415,7 +415,7 @@ gets the time a host is banned for .TP \fBget datepattern\fR -gets the patern used to match +gets the pattern used to match date/times for .TP \fBget usedns\fR diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 8ff407f0..052fce80 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -487,7 +487,7 @@ is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. T .IP \fI\fR - helper regex to match CIDR (simple integer form of net-mask). .IP -\fI\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +\fI\fR - regex to match sub-net addresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). .IP \fI...\fR - free regex capturing group targeting identifier used for ban (instead of IP address or hostname). .IP @@ -540,7 +540,7 @@ There are several prefixes and words with special meaning that could be specifie .IP \fI{UNB}\fR - prefix to disable automatic word boundaries in regex. .IP -\fI{NONE}\fR - value would allow to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). +\fI{NONE}\fR - value would allow one to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). .RE .TP \fBjournalmatch\fR From 3245b8018b2606a5278df42790fac06d5a455f00 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 23 Oct 2021 17:38:20 +0200 Subject: [PATCH 128/139] Add the Debian path to roundcube error logs --- config/paths-debian.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/paths-debian.conf b/config/paths-debian.conf index e096f972..1f5ea37d 100644 --- a/config/paths-debian.conf +++ b/config/paths-debian.conf @@ -26,3 +26,5 @@ exim_main_log = /var/log/exim4/mainlog # was in debian squeezy but not in wheezy # /etc/proftpd/proftpd.conf (SystemLog) proftpd_log = /var/log/proftpd/proftpd.log + +roundcube_errors_log = /var/log/roundcube/errors.log From 4b54a07d71a6ce1c85a3eae92bace6c0dadcdcfb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 1 Nov 2021 11:43:54 +0100 Subject: [PATCH 129/139] Revert "`action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-`;" This reverts the incompatibility #3047 introduced by commit a038fd5dfe8cb0714472833604735b83462a217d (#2821). --- config/action.d/firewallcmd-ipset.conf | 2 +- config/action.d/firewallcmd-multiport.conf | 4 +-- config/action.d/firewallcmd-new.conf | 4 +-- config/action.d/firewallcmd-rich-rules.conf | 4 +-- fail2ban/tests/servertestcase.py | 32 ++++++++++----------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 66358f23..42513933 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -69,7 +69,7 @@ allports = -p # Option: multiport # Notes.: addition to block access only to specific ports # Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -multiport = -p -m multiport --dports "$(echo '' | sed s/:/-/g)" +multiport = -p -m multiport --dports ipmset = f2b- familyopt = diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf index 0c401f1b..81540e5b 100644 --- a/config/action.d/firewallcmd-multiport.conf +++ b/config/action.d/firewallcmd-multiport.conf @@ -11,9 +11,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- + firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index 7b08603c..b06f5ccd 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -10,9 +10,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- + firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index 803d7d12..75a27d88 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -37,8 +37,8 @@ actioncheck = fwcmd_rich_rule = rule family='' source address='' port port='$p' protocol='' %(rich-suffix)s -actionban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done +actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done -actionunban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done +actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done rich-suffix = \ No newline at end of file diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index fc505552..6cf45f53 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1793,18 +1793,18 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", + "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", ), 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", + "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", ), 'stop': ( - """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", + "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`", - """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", + "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`", ), @@ -1872,21 +1872,21 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", - """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", + "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", - """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", + "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset flush f2b-j-w-fwcmd-ipset6`", ), 'stop': ( - """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", + "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset destroy f2b-j-w-fwcmd-ipset`", - """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", + "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", "`ipset flush f2b-j-w-fwcmd-ipset6`", "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), @@ -1943,32 +1943,32 @@ class ServerConfigReaderTests(LogCaptureTestCase): ('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', { 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), 'ip4-ban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", ), 'ip4-unban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", ), 'ip6-ban': ( - """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + """ `ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", ), 'ip6-unban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", ), }), # firewallcmd-rich-logging -- ('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', { 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), 'ip4-ban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", ), 'ip4-unban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", ), 'ip6-ban': ( - """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + """ `ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", ), 'ip6-unban': ( - """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", ), }), ) From 7678f5982733f1d20bbec8d04006ac78cded10db Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Sep 2021 20:42:51 +0200 Subject: [PATCH 130/139] better format of time delta (using seconds2str); increase stability for systemd test-cases --- fail2ban/server/filter.py | 7 ++--- fail2ban/server/mytime.py | 48 ++++++++++++++++++++++++++++++++ fail2ban/tests/filtertestcase.py | 8 ++++-- fail2ban/tests/misctestcase.py | 12 ++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a13b1fb5..d2cd0ce9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -664,13 +664,12 @@ class Filter(JailThread): # if weird date - we'd simulate now for timeing issue (too large deviation from now): delta = int(date - MyTime.time()) if abs(delta) > 60: - delta //= 60 # log timing issue as warning once per day: self._logWarnOnce("_next_simByTimeWarn", - ("Detected a log entry %sm %s the current time in operation mode. " + ("Detected a log entry %s %s the current time in operation mode. " "This looks like a %s problem. Treating such entries as if they just happened.", - abs(delta), "before" if delta < 0 else "after", - "latency" if -55 <= delta < 0 else "timezone" + MyTime.seconds2str(abs(delta)), "before" if delta < 0 else "after", + "latency" if -3300 <= delta < 0 else "timezone" ), ("Please check a jail for a timing issue. Line with odd timestamp: %s", line)) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 49199887..d09c9d41 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -161,3 +161,51 @@ class MyTime: val = rexp.sub(rpl, val) val = MyTime._str2sec_fini.sub(r"\1+\2", val) return eval(val) + + class seconds2str(): + """Converts seconds to string on demand (if string representation needed). + Ex: seconds2str(86400*390) = 1y 3w 4d + seconds2str(86400*368) = 1y 3d + seconds2str(86400*365.5) = 1y + seconds2str(86400*2+3600*7+60*15) = 2d 7h 15m + seconds2str(86400*2+3599) = 2d 1h + seconds2str(3600-5) = 1h + seconds2str(3600-10) = 59m 50s + seconds2str(59) = 59s + """ + def __init__(self, sec): + self.sec = sec + def __str__(self): + # s = str(datetime.timedelta(seconds=int(self.sec))) + # return s if s[-3:] != ":00" else s[:-3] + s = self.sec; r = ""; c = 3 + # automatic accuracy: round by large values (upto 1 minute, or 1 day by year): + if s >= 3570: + if s >= 31536000: + s = int(round(float(s)/86400)*86400) + elif s >= 86400: + s = int(round(float(s)/60)*60) + else: + s = int(round(float(s)/10)*10) + for n, m in ( + ('y', 31536000), # a year as 365*24*60*60 (don't need to consider leap year by this accuracy) + ('w', 604800), # a week as 24*60*60*7 + ('d', 86400), # a day as 24*60*60 + ('h', 3600), # a hour as 60*60 + ('m', 60), # a minute + ('s', 1) # a second + ): + if s >= m: + c -= 1 + r += ' ' + str(s//m) + n + s %= m + # stop if no remaining part or no repeat needed (too small granularity): + if not s or not c: break + elif c < 3: + # avoid too small granularity: + c -= 1 + if not c: break + # + return r[1:] + def __repr__(self): + return self.__str__() diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2b3dd995..343cfe04 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -457,7 +457,7 @@ class IgnoreIP(LogCaptureTestCase): for i in (1,2,3): self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 self.assertLogged( - "Detected a log entry 60m before the current time in operation mode. This looks like a timezone problem.", + "Detected a log entry 1h before the current time in operation mode. This looks like a timezone problem.", "Please check a jail for a timing issue.", "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", "Total # of detected failures: 3.", all=True, wait=True) @@ -467,7 +467,7 @@ class IgnoreIP(LogCaptureTestCase): for i in (1,2,3): self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 self.assertLogged( - "Detected a log entry 120m after the current time in operation mode. This looks like a timezone problem.", + "Detected a log entry 2h after the current time in operation mode. This looks like a timezone problem.", "Please check a jail for a timing issue.", "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", "Total # of detected failures: 6.", all=True, wait=True) @@ -1458,7 +1458,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover if idle: self.filter.sleeptime /= 100.0 self.filter.idle = True - self.waitForTicks(1) + self.waitForTicks(1) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file @@ -1540,6 +1540,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def test_delJournalMatch(self): self._initFilter() self.filter.start() + self.waitForTicks(1); # wait for start # Smoke test for removing of match # basic full test @@ -1572,6 +1573,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def test_WrongChar(self): self._initFilter() self.filter.start() + self.waitForTicks(1); # wait for start # Now let's feed it with entries from the file _copy_lines_to_journal( self.test_file, self.journal_fields, skip=15, n=4) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 458e9a23..915ce7df 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -457,3 +457,15 @@ class MyTimeTest(unittest.TestCase): self.assertEqual(float(str2sec("1 month")) / 60 / 60 / 24, 30.4375) self.assertEqual(float(str2sec("1 year")) / 60 / 60 / 24, 365.25) + def testSec2Str(self): + sec2str = lambda s: str(MyTime.seconds2str(s)) + self.assertEqual(sec2str(86400*390), '1y 3w 4d') + self.assertEqual(sec2str(86400*368), '1y 3d') + self.assertEqual(sec2str(86400*365.49), '1y') + self.assertEqual(sec2str(86400*15), '2w 1d') + self.assertEqual(sec2str(86400*14-10), '2w') + self.assertEqual(sec2str(86400*2+3600*7+60*15), '2d 7h 15m') + self.assertEqual(sec2str(86400*2+3599), '2d 1h') + self.assertEqual(sec2str(3600-5), '1h') + self.assertEqual(sec2str(3600-10), '59m 50s') + self.assertEqual(sec2str(59), '59s') From 96661f25ab558b92b42d01c87d383664e69dd18b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Sep 2021 04:07:55 +0200 Subject: [PATCH 131/139] filtersystemd.py: fixes wrong time point of "in operation" mode todo: need more tests to cover any step of switch to inOperationMode (all branches) --- .github/workflows/main.yml | 5 ++- fail2ban/server/filtersystemd.py | 76 ++++++++++++++++++++++++-------- fail2ban/tests/filtertestcase.py | 23 ++++++++++ 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c60ab05c..e346635d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - + + - name: Grant systemd-journal access + run: sudo usermod -a -G systemd-journal "$USER" || echo 'no systemd-journal access' + - name: Python version run: | F2B_PY=$(python -c "import sys; print(sys.version)") diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 88f8c292..a305b6cf 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -92,8 +92,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover try: args['flags'] = int(kwargs.pop('journalflags')) except KeyError: - # be sure all journal types will be opened if files specified (don't set flags): - if 'files' not in args or not len(args['files']): + # be sure all journal types will be opened if files/path specified (don't set flags): + if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']): args['flags'] = 4 try: @@ -258,6 +258,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover date = datetime.datetime.fromtimestamp(date) self.__journal.seek_realtime(date) + def inOperationMode(self): + self.inOperation = True + logSys.info("[%s] Jail is in operation now (process new journal entries)", self.jailName) + ## # Main loop. # @@ -268,23 +272,44 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if not self.getJournalMatch(): logSys.notice( - "Jail started without 'journalmatch' set. " + "[%s] Jail started without 'journalmatch' set. " "Jail regexs will be checked against all journal entries, " - "which is not advised for performance reasons.") + "which is not advised for performance reasons.", self.jailName) - # Try to obtain the last known time (position of journal) - start_time = 0 - if self.jail.database is not None: - start_time = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0 - # Seek to max(last_known_time, now - findtime) in journal - start_time = max( start_time, MyTime.time() - int(self.getFindTime()) ) - self.seekToTime(start_time) - # Move back one entry to ensure do not end up in dead space - # if start time beyond end of journal + # Save current cursor position (to recognize in operation mode): + logentry = None try: - self.__journal.get_previous() + self.__journal.seek_tail() + logentry = self.__journal.get_previous() + self.__journal.get_next() except OSError: - pass # Reading failure, so safe to ignore + logentry = None # Reading failure, so safe to ignore + if logentry: + # Try to obtain the last known time (position of journal) + startTime = 0 + if self.jail.database is not None: + startTime = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0 + # Seek to max(last_known_time, now - findtime) in journal + startTime = max( startTime, MyTime.time() - int(self.getFindTime()) ) + self.seekToTime(startTime) + # Not in operation while we'll read old messages ... + self.inOperation = False + # Save current time in order to check time to switch "in operation" mode + startTime = (1, MyTime.time(), logentry.get('__CURSOR')) + # Move back one entry to ensure do not end up in dead space + # if start time beyond end of journal + try: + self.__journal.get_previous() + except OSError: + pass # Reading failure, so safe to ignore + else: + # empty journal or no entries for current filter: + self.inOperationMode() + # seek_tail() seems to have a bug by no entries (could bypass some entries hereafter), so seek to now instead: + startTime = MyTime.time() + self.seekToTime(startTime) + # for possible future switches of in-operation mode: + startTime = (0, startTime) line = None while self.active: @@ -317,12 +342,27 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) self.ticks += 1 if logentry: - line = self.formatJournalEntry(logentry) - self.processLineAndAdd(*line) + line, tm = self.formatJournalEntry(logentry) + # switch "in operation" mode if we'll find start entry (+ some delta): + if not self.inOperation: + if tm >= MyTime.time() - 1: # reached now (approximated): + self.inOperationMode() + elif startTime[0] == 1: + # if it reached start entry (or get read time larger than start time) + if logentry.get('__CURSOR') == startTime[2] or tm > startTime[1]: + # give the filter same time it needed to reach the start entry: + startTime = (0, MyTime.time()*2 - startTime[1]) + elif tm > startTime[1]: # reached start time (approximated): + self.inOperationMode() + # process line + self.processLineAndAdd(line, tm) self.__modified += 1 if self.__modified >= 100: # todo: should be configurable break else: + # "in operation" mode since we don't have messages anymore (reached end of journal): + if not self.inOperation: + self.inOperationMode() break self.__modified = 0 if self.ticks % 10 == 0: @@ -334,7 +374,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover or not self.active ) ): - self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.jail.database.updateJournal(self.jail, 'systemd-journal', tm, line[1]) self.__nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 line = None except Exception as e: # pragma: no cover diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 343cfe04..32dcca5c 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1537,6 +1537,29 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover _gen_falure("192.0.2.6") self.assertFalse(self.jail.getFailTicket()) + # now reset DB, so we'd find all messages before filter entering in operation mode: + self.filter.stop() + self.filter.join() + self.jail.database.updateJournal(self.jail, 'systemd-journal', MyTime.time()-10000, 'TEST') + self._initFilter() + self.filter.setMaxRetry(1) + states = [] + def _state(*args): + self.assertNotIn("** in operation", states) + self.assertFalse(self.filter.inOperation) + states.append("** process line: %r" % (args,)) + self.filter.processLineAndAdd = _state + def _inoper(): + self.assertNotIn("** in operation", states) + self.assertEqual(len(states), 11) + states.append("** in operation") + self.filter.__class__.inOperationMode(self.filter) + self.filter.inOperationMode = _inoper + self.filter.start() + self.waitForTicks(12) + self.assertTrue(Utils.wait_for(lambda: len(states) == 12, _maxWaitTime(10))) + self.assertEqual(states[-1], "** in operation") + def test_delJournalMatch(self): self._initFilter() self.filter.start() From 3b020988172c4c38d9baae36344b0d3cfa50332e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Nov 2021 15:35:59 +0100 Subject: [PATCH 132/139] several backends optimizations (in file and journal filters): - don't need to wait if we still had log-entries from last iteration (which got interrupted for servicing) - rewritten update log/journal position, it is more stable and faster now (fewer DB access and surely up-to-date at end) --- fail2ban/server/filter.py | 53 +++++++++++++++++++++++++++----- fail2ban/server/filtersystemd.py | 45 +++++++++++++++++++-------- fail2ban/server/jailthread.py | 8 +++++ 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d2cd0ce9..f8417d2d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -104,6 +104,10 @@ class Filter(JailThread): ## Error counter (protected, so can be used in filter implementations) ## if it reached 100 (at once), run-cycle will go idle self._errors = 0 + ## Next time to update log or journal position in database: + self._nextUpdateTM = 0 + ## Pending updates (must be executed at next update time or during stop): + self._pendDBUpdates = {} ## return raw host (host is not dns): self.returnRawHost = False ## check each regex (used for test purposes): @@ -1023,9 +1027,6 @@ class FileFilter(Filter): log = self.__logs.pop(path) except KeyError: return - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) logSys.info("Removed logfile: %r", path) self._delLogPath(path) return @@ -1145,9 +1146,15 @@ class FileFilter(Filter): self.processLineAndAdd(line.rstrip('\r\n')) finally: log.close() - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) + if self.jail.database is not None: + self._pendDBUpdates[log] = 1 + if ( + self.ticks % 100 == 0 + or MyTime.time() >= self._nextUpdateTM + or not self.active + ): + self._updateDBPending() + self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 return True ## @@ -1247,12 +1254,33 @@ class FileFilter(Filter): ret.append(("File list", path)) return ret - def stop(self): - """Stop monitoring of log-file(s) + def _updateDBPending(self): + """Apply pending updates (log position) to database. """ + db = self.jail.database + while True: + try: + log, args = self._pendDBUpdates.popitem() + except KeyError: + break + db.updateLog(self.jail, log) + + def onStop(self): + """Stop monitoring of log-file(s). Invoked after run method. + """ + # ensure positions of pending logs are up-to-date: + if self._pendDBUpdates and self.jail.database: + self._updateDBPending() # stop files monitoring: for path in self.__logs.keys(): self.delLogPath(path) + + def stop(self): + """Stop filter + """ + # normally onStop will be called automatically in thread after its run ends, + # but for backwards compatibilities we'll invoke it in caller of stop method. + self.onStop() # stop thread: super(Filter, self).stop() @@ -1304,6 +1332,15 @@ class FileContainer: ## shows that log is in operation mode (expecting new messages only from here): self.inOperation = tail + def __hash__(self): + return hash(self.__filename) + def __eq__(self, other): + return (id(self) == id(other) or + self.__filename == (other.__filename if isinstance(other, FileContainer) else other) + ) + def __repr__(self): + return 'file-log:'+self.__filename + def getFileName(self): return self.__filename diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index a305b6cf..e2220105 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -61,7 +61,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # Initialise systemd-journal connection self.__journal = journal.Reader(**jrnlargs) self.__matches = [] - self.__nextUpdateTM = 0 self.setDatePattern(None) logSys.debug("Created FilterSystemd") @@ -321,9 +320,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover #self.__journal.wait(self.sleeptime) != journal.NOP ## ## wait for entries without sleep in intervals, because "sleeping" in journal.wait: - Utils.wait_for(lambda: not self.active or \ - self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP, - self.sleeptime, 0.00001) + if not logentry: + Utils.wait_for(lambda: not self.active or \ + self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP, + self.sleeptime, 0.00001) if self.idle: # because journal.wait will returns immediatelly if we have records in journal, # just wait a little bit here for not idle, to prevent hi-load: @@ -368,15 +368,17 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if self.ticks % 10 == 0: self.performSvc() # update position in log (time and iso string): - if (line and self.jail.database and ( - self.ticks % 10 == 0 - or MyTime.time() >= self.__nextUpdateTM + if self.jail.database: + if line: + self._pendDBUpdates['systemd-journal'] = (tm, line[1]) + line = None + if self._pendDBUpdates and ( + self.ticks % 100 == 0 + or MyTime.time() >= self._nextUpdateTM or not self.active - ) - ): - self.jail.database.updateJournal(self.jail, 'systemd-journal', tm, line[1]) - self.__nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 - line = None + ): + self._updateDBPending() + self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break @@ -403,3 +405,22 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover ret.append(("Journal matches", [" + ".join(" ".join(match) for match in self.__matches)])) return ret + + def _updateDBPending(self): + """Apply pending updates (jornal position) to database. + """ + db = self.jail.database + while True: + try: + log, args = self._pendDBUpdates.popitem() + except KeyError: + break + db.updateJournal(self.jail, log, *args) + + def onStop(self): + """Stop monitoring of journal. Invoked after run method. + """ + # ensure positions of pending logs are up-to-date: + if self._pendDBUpdates and self.jail.database: + self._updateDBPending() + diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 94f34542..67955a06 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -67,6 +67,8 @@ class JailThread(Thread): def run_with_except_hook(*args, **kwargs): try: run(*args, **kwargs) + # call on stop callback to do some finalizations: + self.onStop() except Exception as e: # avoid very sporadic error "'NoneType' object has no attribute 'exc_info'" (https://bugs.python.org/issue7336) # only extremely fast systems are affected ATM (2.7 / 3.x), if thread ends nothing is available here. @@ -97,6 +99,12 @@ class JailThread(Thread): self.active = True super(JailThread, self).start() + @abstractmethod + def onStop(self): # pragma: no cover - absract + """Abstract - Called when thread ends (after run). + """ + pass + def stop(self): """Sets `active` property to False, to flag run method to return. """ From 4fe4ac8dde6ba14841da598ec37f8c6911fe0f64 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Nov 2021 15:58:57 +0100 Subject: [PATCH 133/139] amend to merge: replace timedelta string representation with new function seconds2str --- fail2ban/server/observer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index f5ba20d9..b585706f 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -462,7 +462,7 @@ class ObserverThread(JailThread): if ticket.getTime() > timeOfBan: logSys.info('[%s] IP %s is bad: %s # last %s - incr %s to %s' % (jail.name, ip, banCount, MyTime.time2str(timeOfBan), - datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + MyTime.seconds2str(orgBanTime), MyTime.seconds2str(banTime))) else: ticket.restored = True break @@ -491,8 +491,7 @@ class ObserverThread(JailThread): # if not permanent if btime != -1: bendtime = ticket.getTime() + btime - logtime = (datetime.timedelta(seconds=int(btime)), - MyTime.time2str(bendtime)) + logtime = (MyTime.seconds2str(btime), MyTime.time2str(bendtime)) # check ban is not too old : if bendtime < MyTime.time(): logSys.debug('Ignore old bantime %s', logtime[1]) From a57643404c98fee74398cb175e7de1d3883a6f99 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 4 Nov 2021 14:34:04 +0100 Subject: [PATCH 134/139] mytime.seconds2str: small amend with speed-up, code simplification and few tests --- fail2ban/server/mytime.py | 69 ++++++++++++++++++++-------------- fail2ban/tests/misctestcase.py | 3 ++ 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index d09c9d41..b6770721 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -178,34 +178,45 @@ class MyTime: def __str__(self): # s = str(datetime.timedelta(seconds=int(self.sec))) # return s if s[-3:] != ":00" else s[:-3] - s = self.sec; r = ""; c = 3 - # automatic accuracy: round by large values (upto 1 minute, or 1 day by year): - if s >= 3570: - if s >= 31536000: - s = int(round(float(s)/86400)*86400) - elif s >= 86400: - s = int(round(float(s)/60)*60) - else: - s = int(round(float(s)/10)*10) - for n, m in ( - ('y', 31536000), # a year as 365*24*60*60 (don't need to consider leap year by this accuracy) - ('w', 604800), # a week as 24*60*60*7 - ('d', 86400), # a day as 24*60*60 - ('h', 3600), # a hour as 60*60 - ('m', 60), # a minute - ('s', 1) # a second - ): - if s >= m: - c -= 1 - r += ' ' + str(s//m) + n - s %= m - # stop if no remaining part or no repeat needed (too small granularity): - if not s or not c: break - elif c < 3: - # avoid too small granularity: - c -= 1 - if not c: break - # - return r[1:] + s = self.sec; c = 3 + # automatic accuracy: round by large values (and maximally 3 groups) + if s >= 31536000: # a year as 365*24*60*60 (don't need to consider leap year by this accuracy) + s = int(round(float(s)/86400)) # round by a day + r = str(s//365) + 'y '; s %= 365 + if s >= 7: + r += str(s//7) + 'w '; s %= 7 + if s: + r += str(s) + 'd ' + return r[:-1] + if s >= 604800: # a week as 24*60*60*7 + s = int(round(float(s)/3600)) # round by a hour + r = str(s//168) + 'w '; s %= 168 + if s >= 24: + r += str(s//24) + 'd '; s %= 24 + if s: + r += str(s) + 'h ' + return r[:-1] + if s >= 86400: # a day as 24*60*60 + s = int(round(float(s)/60)) # round by a minute + r = str(s//1440) + 'd '; s %= 1440 + if s >= 60: + r += str(s//60) + 'h '; s %= 60 + if s: + r += str(s) + 'm ' + return r[:-1] + if s >= 3595: # a hour as 60*60 (- 5 seconds) + s = int(round(float(s)/10)) # round by 10 seconds + r = str(s//360) + 'h '; s %= 360 + if s >= 6: # a minute + r += str(s//6) + 'm '; s %= 6 + return r[:-1] + r = '' + if s >= 60: # a minute + r += str(s//60) + 'm '; s %= 60 + if s: # remaining seconds + r += str(s) + 's ' + elif not self.sec: # 0s + r = '0 ' + return r[:-1] def __repr__(self): return self.__str__() diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 915ce7df..97669be0 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -466,6 +466,9 @@ class MyTimeTest(unittest.TestCase): self.assertEqual(sec2str(86400*14-10), '2w') self.assertEqual(sec2str(86400*2+3600*7+60*15), '2d 7h 15m') self.assertEqual(sec2str(86400*2+3599), '2d 1h') + self.assertEqual(sec2str(3600*3.52), '3h 31m') + self.assertEqual(sec2str(3600*2-5), '2h') self.assertEqual(sec2str(3600-5), '1h') self.assertEqual(sec2str(3600-10), '59m 50s') self.assertEqual(sec2str(59), '59s') + self.assertEqual(sec2str(0), '0') From 1bcb62e31ca0372abc4fefb81724f30fb6ff8837 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Nov 2021 19:15:59 +0100 Subject: [PATCH 135/139] gh-actions: python releases upgrade + debug/test systemd backend availability in GHA --- .github/workflows/main.yml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e346635d..c0dc726b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-rc.2', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -40,33 +40,44 @@ jobs: - name: Python version run: | F2B_PY=$(python -c "import sys; print(sys.version)") - echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }" F2B_PY=${F2B_PY:0:1} echo "Set F2B_PY=$F2B_PY" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV - name: Install dependencies run: | - #python -m pip install --upgrade pip + if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then #pip install 2to3 sudo apt-get -y install 2to3 fi - #pip install systemd-python || echo 'systemd not available' - #pip install pyinotify || echo 'inotify not available' - sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' - sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' + #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' + python -m pip install pyinotify || echo 'inotify not available' + #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' + sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable' + python -m pip install systemd-python || echo 'systemd not available' - name: Before scripts run: | cd "$GITHUB_WORKSPACE" # Manually execute 2to3 for now if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; } # (debug) output current preferred encoding: - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + # (debug) backend availabilities: + echo 'Backends:' + _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - - name: Test suite - run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + # - name: Test suite + # run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + - name: Test suite (debug some systemd tests only) + # run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" + run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" - name: Build run: python setup.py build From a147a8b0e1b2f32b6f191932afd3c2db9765e2e3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 1 Dec 2021 20:04:46 +0100 Subject: [PATCH 136/139] gh-actions: coverage for systemd backend (to monitor journals in test-suite in GHA-env we need to use 0 as default flags, because otherwise it cannot be found using SYSTEM_ONLY(4)) --- .github/workflows/main.yml | 14 ++++++++------ fail2ban/server/filtersystemd.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0dc726b..788b17e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,8 @@ jobs: F2B_PY=${F2B_PY:0:1} echo "Set F2B_PY=$F2B_PY" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4): + echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV - name: Install dependencies run: | @@ -71,13 +73,13 @@ jobs: _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd' _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - - # - name: Test suite - # run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi - - name: Test suite (debug some systemd tests only) - # run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" - run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test suite (debug some systemd tests only) + #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" + #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" - name: Build run: python setup.py build diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index e2220105..6301b93a 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import datetime +import os import time from distutils.version import LooseVersion @@ -93,7 +94,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover except KeyError: # be sure all journal types will be opened if files/path specified (don't set flags): if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']): - args['flags'] = 4 + args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4)) try: args['namespace'] = kwargs.pop('namespace') From 0fa76ef75a7c6102f383c703fdcf6d53e45149b0 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 2 Dec 2021 18:27:56 +0100 Subject: [PATCH 137/139] gh-actions: temporary ignore tests of systemd backend for python >= v.3.10 (otherwise it fails with "PY_SSIZE_T_CLEAN macro must be defined for '#' formats") --- .github/workflows/main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 788b17e3..ff31db19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,9 +41,11 @@ jobs: run: | F2B_PY=$(python -c "import sys; print(sys.version)") echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }" + F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)') F2B_PY=${F2B_PY:0:1} - echo "Set F2B_PY=$F2B_PY" + echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4): echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV @@ -75,7 +77,15 @@ jobs: _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - name: Test suite - run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + run: | + if [[ "$F2B_PY" = 2 ]]; then + python setup.py test + elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then + python bin/fail2ban-testcases --verbosity=2 + else + echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." + python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" + fi #- name: Test suite (debug some systemd tests only) #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" From 196c55e93103d1e56d1336f27a5f2591b4e54fef Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:49:18 +0200 Subject: [PATCH 138/139] fix gh-3098: build fails with error in fail2ban setup command: use_2to3 is invalid (setuptools 58+) --- setup.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/setup.py b/setup.py index f4c2550f..98413273 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ import warnings from glob import glob from fail2ban.setup import updatePyExec - +from fail2ban.version import version source_dir = os.path.realpath(os.path.dirname( # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): @@ -112,22 +112,12 @@ class install_scripts_f2b(install_scripts): # Wrapper to specify fail2ban own options: class install_command_f2b(install): user_options = install.user_options + [ - ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): - self.disable_2to3 = None self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): - global _2to3 - ## in the test cases 2to3 should be already done (fail2ban-2to3): - if self.disable_2to3: - _2to3 = False - if _2to3: - cmdclass = self.distribution.cmdclass - cmdclass['build_py'] = build_py_2to3 - cmdclass['build_scripts'] = build_scripts_2to3 if self.without_tests: self.distribution.scripts.remove('bin/fail2ban-testcases') @@ -178,7 +168,6 @@ commands.''' if setuptools: setup_extra = { 'test_suite': "fail2ban.tests.utils.gatherTests", - 'use_2to3': True, } else: setup_extra = {} @@ -202,9 +191,6 @@ if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswit ('/usr/share/doc/fail2ban', doc_files) ) -# Get version number, avoiding importing fail2ban. -# This is due to tests not functioning for python3 as 2to3 takes place later -exec(open(join("fail2ban", "version.py")).read()) setup( name = "fail2ban", From 21d94ff178192116eb5288190d78dd4630534a7b Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:52:34 +0200 Subject: [PATCH 139/139] amend to fix gh-3098: no option `--disable-2to3` anymore --- fail2ban/tests/misctestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 97669be0..e2faa6fd 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -111,7 +111,7 @@ class SetupTest(unittest.TestCase): supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: # try dry-run: - os.system("%s %s --dry-run install --disable-2to3 --root=%s%s" + os.system("%s %s --dry-run install --root=%s%s" % (sys.executable, self.setup , tmp, supdbgout)) # check nothing was created: self.assertTrue(not os.listdir(tmp)) @@ -127,7 +127,7 @@ class SetupTest(unittest.TestCase): # suppress stdout (and stderr) if not heavydebug supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: - self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s" + self.assertEqual(os.system("%s %s install --root=%s%s" % (sys.executable, self.setup, tmp, supdbgout)), 0) def strippath(l):