From 8614ca8c41a0d868f0d54d219666374216fb70d4 Mon Sep 17 00:00:00 2001 From: Shane Forsythe <2287983+shaneforsythe@users.noreply.github.com> Date: Tue, 2 Oct 2018 17:24:33 -0400 Subject: [PATCH 001/185] Update proftpd.conf proftpd 1.3.5e can leave inconsistent error message if ftp or mod_sftp is used Oct 2 15:45:31 ftp01 proftpd[5516]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted Oct 2 15:45:44 ftp01 proftpd[5517]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted. Fix regex to make trailing period optional, otherwise brute force attacks against root account using ftp are not blocked correctly. --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 303be5e5..feb59f11 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -18,7 +18,7 @@ __suffix_failed_login = (User not authorized for login|No such user found|Incorr failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\.? *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ ignoreregex = From 606bf110c99c0b491b10f336b67675311f279f1a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 16 Mar 2020 17:29:09 +0100 Subject: [PATCH 002/185] filter.d/sshd.conf (mode `ddos`): fixed "connection reset" regex (seems to have same syntax now as closed), so both regex's combined now to single RE (closes gh-2662) --- config/filter.d/sshd.conf | 3 +-- .../tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 3 +-- fail2ban/tests/files/logs/sshd | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 12631cb3..7a7f5e48 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -73,11 +73,10 @@ mdre-normal-other = ^(Connection closed|Disconnected) ^kex_exchange_identification: client sent invalid protocol identifier ^Bad protocol version identification '.*' from - ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer # same as mdre-normal-other, but as failure (without ) and [preauth] only: -mdre-ddos-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ +mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index d61a6520..4ff4ac68 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -57,8 +57,7 @@ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from ^%(__prefix_line_sl)sBad protocol version identification '.*' from - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 0385f38c..e45ca90d 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -296,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } From 343ec1cdd296530f331637c725bd2bb0549e01e6 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 18 Mar 2020 20:37:25 +0100 Subject: [PATCH 003/185] test-causes: avoid host-depending issue (mistakenly ignoring IP 127.0.0.2 as own address) - replace loop-back addr with test sub-net addr (and disable ignoreself) --- fail2ban/tests/observertestcase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 8e944454..e379ccd1 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -224,7 +223,7 @@ class BanTimeIncrDB(LogCaptureTestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -385,10 +384,12 @@ class BanTimeIncrDB(LogCaptureTestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -477,7 +478,7 @@ class BanTimeIncrDB(LogCaptureTestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) failManager = FailManager() failManager.setMaxRetry(3) From fc175fa78a2c0c91f6f90745c12a56e67605a279 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:10:32 +0200 Subject: [PATCH 004/185] performance: optimize simplest case whether the ignoreip is a single IP (not subnet/dns) - uses a set instead of list (holds single IPs and subnets/dns in different lists); decrease log level for ignored duplicates (warning is too heavy here) --- fail2ban/server/filter.py | 24 ++++++++++++++++++------ fail2ban/server/ipdns.py | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e7f3e01d..a92acb8b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -80,6 +80,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -489,28 +490,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -550,8 +559,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 6648dac6..335fc473 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -379,6 +379,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other From d21a24de8e1bb998fe8a54908b650250ba524fd0 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:39:36 +0200 Subject: [PATCH 005/185] more test cases for IP/DNS (and use dummies if no-network set by testing) --- fail2ban/tests/filtertestcase.py | 16 ++++++++++++---- fail2ban/tests/utils.py | 12 ++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 202f3fbb..a511b5d0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1899,7 +1899,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip4 = IPAddr('192.0.2.1') ip6 = IPAddr('2001:DB8::') self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) self.assertTrue(asip('192.0.2.1').isIPv4) self.assertTrue(id(asip(ip4)) == id(ip4)) @@ -1908,6 +1910,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1916,6 +1919,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1938,7 +1942,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertSortedEqual(res, []) - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1947,7 +1951,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1971,7 +1975,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: @@ -1993,8 +1997,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -2019,6 +2025,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -2114,7 +2122,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index f5ffb978..dc12a5be 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -331,13 +331,21 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), ('192.168.0.', set()), ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): From 136781d627aa70ab88f3fac3b6df398c21dc7387 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Apr 2020 12:17:59 +0200 Subject: [PATCH 006/185] filter.d/sshd.conf: fixed regex for mode `extra` - "No authentication methods available" (supported seems to be optional now, gh-2682) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 7a7f5e48..31e61b96 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -78,7 +78,7 @@ mdre-ddos = ^Did not receive identification string from # same as mdre-normal-other, but as failure (without ) and [preauth] only: mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 4ff4ac68..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -60,7 +60,7 @@ mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e45ca90d..1bf9d913 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -330,6 +330,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } From 2912bc640b3335bffe20d03afa74d84c7a3c4f56 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Thu, 9 Apr 2020 16:42:08 +0200 Subject: [PATCH 007/185] New Gitlab jail --- config/filter.d/gitlab.conf | 6 ++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/gitlab | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 config/filter.d/gitlab.conf create mode 100644 fail2ban/tests/files/logs/gitlab diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=.+ ip=$ diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..e5d16656 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -821,6 +821,10 @@ udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + [bitwarden] port = http,https logpath = /home/*/bwdata/logs/identity/Identity/log.txt diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..222df642 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 78651de7e504bad84e73040dfd7ea530591d0385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 14 Apr 2020 12:25:18 +0200 Subject: [PATCH 008/185] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 3781f467..c722db2b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition should be interpolated in definition section (inside the filter-config, gh-2650) ### New Features +* new filter and jail for GitLab recognizing failed application logins (gh-2689) ### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; From 7e3061e7ace0e973378a17de04a2692142e1a6c1 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Apr 2020 17:35:04 +0200 Subject: [PATCH 009/185] fail2ban.service systemd unit template: don't add user site directory to python system path (avoids accessing of `/root/.local` directory, prevents SE linux audit warning at daemon startup, gh-2688) --- files/fail2ban.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 5e540545..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftabl [Service] Type=simple +Environment="PYTHONNOUSERSITE=1" ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local From 06b46e92eb3fc11c29974fea1753a96edaf8ec20 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Apr 2020 21:34:30 +0200 Subject: [PATCH 010/185] jail.conf: don't specify `action` directly in jails (use `action_` or `banaction` instead); no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified per jail or in default section in jail.local), closes gh-2357; ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686); don't use %(banaction)s interpolation because it can be complex value (containing `[...]`), so would bother the action interpolation. --- ChangeLog | 6 ++++++ config/jail.conf | 30 ++++++++++++++---------------- fail2ban/client/actionreader.py | 10 +++++++--- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3781f467..6329d39a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,12 @@ ver. 0.10.6-dev (20??/??/??) - development edition * python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..bbf8740f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -174,19 +174,19 @@ banaction_allports = iptables-allports action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mwl = %(action_)s %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines @@ -333,7 +333,7 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log @@ -706,8 +706,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -718,9 +718,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -728,9 +727,8 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 @@ -818,8 +816,8 @@ logpath = /opt/cstrike/logs/L[0-9]*.log # Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] [bitwarden] port = http,https @@ -871,8 +869,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index e5bee154..131e37cb 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -52,13 +52,17 @@ class ActionReader(DefinitionInitConfigReader): } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) From affd9cef5f2ddb5c596e1aa3789ba18b5c987ba1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Apr 2020 13:32:17 +0200 Subject: [PATCH 011/185] filter.d/courier-smtp.conf: prefregex extended to consider port in log-message (closes gh-2697) --- ChangeLog | 1 + config/filter.d/courier-smtp.conf | 2 +- fail2ban/tests/files/logs/courier-smtp | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6329d39a..b001ae58 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) ### New Features diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=,.+$ +prefregex = ^%(__prefix_line)serror,relay=,(?:port=\d+,)?.+$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..cea73073 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -12,3 +12,5 @@ Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com From 6b90ca820f5244fd88e8b407419da359d5d068b9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Apr 2020 13:08:24 +0200 Subject: [PATCH 012/185] filter.d/traefik-auth.conf: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle the match of username differently: - `normal`: matches 401 with supplied username only - `ddos`: matches 401 without supplied username only - `aggressive`: matches 401 and any variant (with and without username) closes gh-2693 --- ChangeLog | 5 +++++ config/filter.d/traefik-auth.conf | 22 +++++++++++++++++++++- fail2ban/tests/files/logs/traefik-auth | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b001ae58..f5d3dd6d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,6 +47,11 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) ### New Features diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf index 8321a138..8022fee1 100644 --- a/config/filter.d/traefik-auth.conf +++ b/config/filter.d/traefik-auth.conf @@ -51,6 +51,26 @@ [Definition] -failregex = ^ \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^ \- > \[\] \"(?:) [^\"]+\" 401\b ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )\S+ +usrre-ddos = - +usrre-aggressive = \S+ \ No newline at end of file diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth index 3e7a8987..edfe7306 100644 --- a/fail2ban/tests/files/logs/traefik-auth +++ b/fail2ban/tests/files/logs/traefik-auth @@ -1,6 +1,23 @@ +# filterOptions: [{"mode": "normal"}] + # failJSON: { "match": false } 10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "ddos"}] + +# failJSON: { "match": false } +10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}] + # failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" } 10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms +# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" } +10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms # failJSON: { "match": false } 10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" } +10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms From 87a1a2f1a112ce5bfef8357758435266c4f6b43d Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 14:52:38 +0200 Subject: [PATCH 013/185] action.d/*-ipset*.conf: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only) --- config/action.d/firewallcmd-ipset.conf | 22 +++++++++++------ .../iptables-ipset-proto6-allports.conf | 24 ++++++++++++------- config/action.d/iptables-ipset-proto6.conf | 24 ++++++++++++------- config/action.d/shorewall-ipset-proto6.conf | 22 ++++++++++------- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index dcf20375..9dd9fbb2 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist actionunban = ipset del -exist @@ -40,11 +40,19 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype # Notes.: defines additions to the blocking rule @@ -69,7 +77,7 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 # DEV NOTES: diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index dc7d63a7..4f200db0 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index f88777b8..8956ec6a 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index fc7dd24e..cbcc5524 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -76,10 +76,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -[Init] +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) From 12be3ed77d9cbe0ef6fed4ac26f9859527f9c448 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 15:17:42 +0200 Subject: [PATCH 014/185] test cases fixed --- fail2ban/tests/servertestcase.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index a51c4f85..b771ab50 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1509,14 +1509,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0 `", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1532,27 +1532,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain=""]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1568,13 +1568,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", @@ -1852,14 +1852,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1875,27 +1875,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=, protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=, protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1911,13 +1911,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", From da1652d0d71074a2af8ae69512fbca24fafdf385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 26 Apr 2020 12:26:55 +0200 Subject: [PATCH 015/185] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index f5d3dd6d..2ae56d99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 5da2422f616f6245982f38693df2e61ccf24dbb4 Mon Sep 17 00:00:00 2001 From: Ilya Date: Wed, 11 Mar 2020 14:43:45 +0300 Subject: [PATCH 016/185] Fix actionunban Add command to remove new line character. Needed for working removing rule from cloudflare firewall. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..70e5ee3f 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) [Init] From 8b3b9addd10dec9fec47b8f7fd3a971c326fe430 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 20 Mar 2020 13:52:17 +0300 Subject: [PATCH 017/185] Change tool from 'cut' to 'sed' Sed regex was tested - it works. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 70e5ee3f..d00db98b 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) [Init] From 852670bc99be2f30a64dd5d096d5ff375b6a2e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Sun, 27 May 2018 08:15:33 +0200 Subject: [PATCH 018/185] CloudFlare started to indent their API responses We need to use https://github.com/stedolan/jq to parse it. --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index d00db98b..27a0b6b5 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl and jq. # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') [Init] From 5b8fc3b51a203a7428dd66f07fb8480ec894860d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 Dec 2018 14:35:17 +0100 Subject: [PATCH 019/185] cloudflare: fixes ip to id conversion by unban using jq normalized URIs and parameters, notes gets a jail-name (should be possible to differentiate the same IP across several jails) --- config/action.d/cloudflare.conf | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 27a0b6b5..5125777c 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + | jq -r '.result[0].id') + if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 1c1b671c745dbe0e1f9a096fd1953d0257e8b958 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Dec 2018 11:27:53 +0100 Subject: [PATCH 020/185] Update cloudflare.conf --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 5125777c..4c0a3810 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -59,13 +59,13 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | jq -r '.result[0].id') if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" _cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules -_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 01e92ce4a617ff8cf95d47b67cdd77481a3efdd7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 16 Mar 2020 18:28:45 +0100 Subject: [PATCH 021/185] added fallback using tr and sed (jq is optional now) --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4c0a3810..361cb177 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl and jq. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ - | jq -r '.result[0].id') + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" From 42aef09d695f98794066118015db7b72442c6116 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 27 Apr 2020 19:38:48 +0200 Subject: [PATCH 022/185] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2ae56d99..d744dba0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 43f699b872ab75132a2188be29706d602830254e Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 May 2020 17:32:13 +0200 Subject: [PATCH 023/185] grammar / typos --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c7177f13..4d236b34 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,7 +60,7 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # From afb7a931632cded1312facf2f82d2a33caf7ffd2 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:27:48 +0200 Subject: [PATCH 024/185] amend to 368aa9e77570519b37fb57c9dbc5112d4c4b7382: fix time in gitlab test (GMT in log due to TZ-suffix `Z`, CEST in test-suite) --- fail2ban/tests/files/logs/gitlab | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab index 222df642..70ddc0e8 100644 --- a/fail2ban/tests/files/logs/gitlab +++ b/fail2ban/tests/files/logs/gitlab @@ -1,5 +1,5 @@ # Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log -# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } 2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 -# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } 2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 0ae2ef68be57f056038adaf064570e6725c6f20f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:36:06 +0200 Subject: [PATCH 025/185] ensure iterator is safe (traverse over the list in snapshot created within a lock), avoids getting modified state as well as "dictionary changed size during iteration" errors --- fail2ban/server/banmanager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 8c0a6965..479ba26f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -104,7 +104,7 @@ class BanManager: def getBanList(self): with self.__lock: - return self.__banList.keys() + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,8 +112,9 @@ class BanManager: # @return ban list iterator def __iter__(self): + # ensure iterator is safe (traverse over the list in snapshot created within lock): with self.__lock: - return self.__banList.itervalues() + return iter(list(self.__banList.values())) ## # Returns normalized value From 54b2208690e3c2fff00fbd9b197984d880e29a02 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 16:31:54 +0200 Subject: [PATCH 026/185] extends protocol/client with banned status (retrieve information whether an IP is banned and/or in which jails), implements FR gh-2725 --- fail2ban/protocol.py | 4 +++ fail2ban/server/actions.py | 8 ++++++ fail2ban/server/server.py | 26 ++++++++++++++++++ fail2ban/server/transmitter.py | 8 +++++- fail2ban/tests/fail2banclienttestcase.py | 34 +++++++++++++++++++++++- man/fail2ban-client.1 | 16 +++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 92b0dcc0..18b901e8 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban ... ", "unbans (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned ... ]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -120,6 +122,8 @@ protocol = [ ["set action ", "sets the of for the action for "], ["set action [ ]", "calls the with for the action for "], ['', "JAIL INFORMATION", ""], +["get banned", "return banned IPs of "], +["get banned ... ]", "return 1 if IP is banned in otherwise 0, or a list of 1/0 for multiple IPs"], ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 35b027ae..6123605d 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -210,6 +210,14 @@ class Actions(JailThread, Mapping): def getBanTime(self): return self.__banManager.getBanTime() + def getBanned(self, ids): + lst = self.__banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + def addBannedIP(self, ip): """Ban an IP or list of IPs.""" unixTime = MyTime.time() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index abb312da..b12c8f9f 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -521,6 +521,32 @@ class Server: cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index de80f624..aff9071c 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -118,6 +118,9 @@ class Transmitter: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) elif name == "echo": return command[1:] elif name == "server-status": @@ -424,7 +427,10 @@ class Transmitter: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 6c79800e..a334b568 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -421,6 +421,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -1007,6 +1015,30 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail2] Found 192.0.2.3", "[test-jail2] Ban 192.0.2.3", all=True) + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # rotate logs: _write_file(test1log, "w+") diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 84f846f2..6a3247e1 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -111,6 +111,14 @@ jails and database) unbans (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned ... ]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -356,6 +364,14 @@ for .IP JAIL INFORMATION .TP +\fBget banned\fR +return banned IPs of +.TP +\fBget banned ... ]\fR +return 1 if IP is banned in +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget logpath\fR gets the list of the monitored files for From fa1ff4c5d8757bf8e07bd43794d27290293192e5 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 May 2020 13:36:09 +0200 Subject: [PATCH 027/185] assertSortedEqual: fixed sort of nested lists, switch default of nestedOnly to False (comparison of unsorted lists is rarely needed) --- fail2ban/tests/misctestcase.py | 10 +++++++++- fail2ban/tests/utils.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 9b986f53..43d76802 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -390,7 +390,15 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index dc12a5be..47e5b909 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -556,7 +556,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -566,6 +566,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -584,8 +591,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) From 9b6da03c90139db1a8820a24736462c5c3477ba8 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 26 May 2020 13:14:37 +0200 Subject: [PATCH 028/185] amend to e786dbf132689133c29671871718a97f93b8912a: removes space between name and [pid] by normal non-verbose logging, padding without truncate now; test coverage for getVerbosityFormat; closes #2734 --- fail2ban/helpers.py | 2 +- fail2ban/tests/misctestcase.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 6f2bcdd7..3ef7d543 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -302,7 +302,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True if addtime: fmt = ' %(asctime)-15s' + fmt else: # default (not verbose): - fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt + fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt if addtime: fmt = "%(asctime)s " + fmt # remove padding if not needed: diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 43d76802..c1a6a345 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -34,7 +34,7 @@ from StringIO import StringIO from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ - splitwords, uni_decode, uni_string + getVerbosityFormat, splitwords, uni_decode, uni_string from ..server.mytime import MyTime @@ -404,6 +404,14 @@ class TestsUtilsTest(LogCaptureTestCase): self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", self.assertSortedEqual, ['A', 'B'], ['C', 'B']) + def testVerbosityFormat(self): + self.assertEqual(getVerbosityFormat(1), + '%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s') + self.assertEqual(getVerbosityFormat(1, padding=False), + '%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s') + self.assertEqual(getVerbosityFormat(1, addtime=False, padding=False), + '%(name)s[%(process)d]: %(levelname)s %(message)s') + def testFormatterWithTraceBack(self): strout = StringIO() Formatter = FormatterWithTraceBack From 412120ac3c3f1c9a3fa8c7676059ab2be0f87c6a Mon Sep 17 00:00:00 2001 From: aresdr Date: Sat, 30 May 2020 15:25:31 -0700 Subject: [PATCH 029/185] Update drupal-auth.conf Small fix for Drupal 8. D8 uses "Login attempt failed from" while D7 uses "Login attempt failed for". The referer part is a must currently, but some requests did not have one and are not failing. --- config/filter.d/drupal-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index b60abe3e..2d4cbe9f 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.+\|\d\|.*\|Login attempt failed for .+\.$ +failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.*\|\d\|.*\|Login attempt failed (?:for|from) .+\.$ ignoreregex = From 3c83c19070b8e79e2d4392071d02b4ea99643d87 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sat, 6 Jun 2020 19:51:46 +0200 Subject: [PATCH 030/185] Added filter nginx-bad-request --- config/filter.d/nginx-bad-request.conf | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 config/filter.d/nginx-bad-request.conf diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf new file mode 100644 index 00000000..ea26d56a --- /dev/null +++ b/config/filter.d/nginx-bad-request.conf @@ -0,0 +1,13 @@ +# Fail2Ban filter to match bad requests to nginx +# + +[Definition] + +# The request often doesn't contain a method, only some encoded garbage +failregex = ^ \- \S+ \[\] \".+\" 400 .+$ + +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + ^[^\[]*\[({DATE}) + {^LN-BEG} + +# Author: Jan Przybylak From 5a0edf61c96998a29d4e7afd2105cf8f5b855318 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 8 Jun 2020 14:38:26 +0200 Subject: [PATCH 031/185] filter.d/sshd.conf: normalizing of user pattern in all RE's, allowing empty user (gh-2749) --- ChangeLog | 1 + config/filter.d/sshd.conf | 16 ++++++++-------- fail2ban/tests/files/logs/sshd | 5 +++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index c5be981b..013cba89 100644 --- a/ChangeLog +++ b/ChangeLog @@ -56,6 +56,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition - `normal`: matches 401 with supplied username only - `ddos`: matches 401 without supplied username only - `aggressive`: matches 401 and any variant (with and without username) +* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) ### New Features * new filter and jail for GitLab recognizing failed application logins (gh-2689) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 31e61b96..4c86dca0 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* __on_port_opt = (?: (?:port \d+|on \S+)){0,2} # close by authenticating user: -__authng_user = (?: (?:invalid|authenticating) user \S+|.+?)? +__authng_user = (?: (?:invalid|authenticating) user \S+|.*?)? # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. @@ -44,18 +44,18 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^Failed for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^ROOT LOGIN REFUSED FROM ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ - ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ - ^User .+ from not allowed because listed in DenyUsers%(__suff)s$ - ^User .+ from not allowed because not in any group%(__suff)s$ + ^User \S+|.*? from not allowed because not listed in AllowUsers%(__suff)s$ + ^User \S+|.*? from not allowed because listed in DenyUsers%(__suff)s$ + ^User \S+|.*? from not allowed because not in any group%(__suff)s$ ^refused connect from \S+ \(\) ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ - ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ + ^User \S+|.*? from not allowed because a group is listed in DenyGroups%(__suff)s$ + ^User \S+|.*? from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ ^maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ - ^User .+ not allowed because account is locked%(__suff)s + ^User \S+|.*? not allowed because account is locked%(__suff)s ^Disconnecting(?: from)?(?: (?:invalid|authenticating)) user \S+ %(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for \S+|.*?)?%(__suff)s$ ^Received disconnect from %(__on_port_opt)s:\s*11: -other> ^Accepted \w+ for \S+ from (?:\s|$) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 1bf9d913..9fff416a 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -321,6 +321,11 @@ Mar 15 09:21:02 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-07-18T17:19:11", "match": true , "host": "192.0.2.4", "desc": "ddos: disconnect on preauth phase, gh-2115" } Jul 18 17:19:11 srv sshd[2101]: Disconnected from 192.0.2.4 port 36985 [preauth] +# failJSON: { "time": "2005-06-06T04:17:04", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:04 host sshd[1189074]: Invalid user from 192.0.2.68 port 34916 +# failJSON: { "time": "2005-06-06T04:17:09", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:09 host sshd[1189074]: Connection closed by invalid user 192.0.2.68 port 34916 [preauth] + # filterOptions: [{"mode": "extra"}, {"mode": "aggressive"}] # several other cases from gh-864: From d7ef5d166db58ec402408fa97c0b291906c3d8c9 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Thu, 11 Jun 2020 16:44:48 +0200 Subject: [PATCH 032/185] Removed vulnerable catchall & anchor --- config/filter.d/nginx-bad-request.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index ea26d56a..03721f86 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -4,7 +4,7 @@ [Definition] # The request often doesn't contain a method, only some encoded garbage -failregex = ^ \- \S+ \[\] \".+\" 400 .+$ +failregex = ^ \- \S+ \[\] \"[^\"]+\" 400 datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? ^[^\[]*\[({DATE}) From dd8081ade5ace23782efcf0c96b7f8c9f16228a7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 20:00:42 +0200 Subject: [PATCH 033/185] extends capturing alternate tags in filter, implementing new tag prefix `` with all value of tags), for examples see new tests in fail2banregextestcase; closes gh-2755 (extends #1454 and #1698). --- fail2ban/server/action.py | 2 +- fail2ban/server/failregex.py | 56 +++++++++++++++++-------- fail2ban/server/ipdns.py | 2 +- fail2ban/tests/fail2banregextestcase.py | 17 ++++++++ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d0262171..1c313cf0 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -868,7 +868,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 0ae9acc5..9bbf5779 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -87,20 +87,24 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -128,18 +132,23 @@ class Regex: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex self._altValues = {} + self._tupleValues = {} for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n + n = COMPLNAME_CRE.match(k) + if n: + if n.group(1) == ALTNAME_PRE: + self._altValues[k] = mapTag2Opt(n.group(2)) + else: + self._tupleValues[k] = mapTag2Opt(n.group(2)) self._altValues = list(self._altValues.items()) if len(self._altValues) else None + self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) # set fetch handler depending on presence of alternate tags: - self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -284,12 +293,23 @@ class Regex: def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..18e1bd02 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -337,7 +337,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..334fad1c 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -340,6 +340,23 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) self.assertLogged('kevin') self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s* \S+')) + self.assertLogged(str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left 192.0.2.3 right', + r'^\s*\S+ \S+')) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*\S+ : \S+')) + self.assertLogged(str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() # row with id : self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) self.assertLogged("['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) From ec3000798d9b42c3f05d40e810f43ae444c14d97 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 21:25:42 +0200 Subject: [PATCH 034/185] ensure that set of alternate tags or combine tuple tags take place ordered (sort the lists by its name or index) --- fail2ban/server/failregex.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 9bbf5779..4032ecdb 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -131,23 +131,26 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} - self._tupleValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): n = COMPLNAME_CRE.match(k) if n: - if n.group(1) == ALTNAME_PRE: - self._altValues[k] = mapTag2Opt(n.group(2)) + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) else: - self._tupleValues[k] = mapTag2Opt(n.group(2)) - self._altValues = list(self._altValues.items()) if len(self._altValues) else None - self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - # set fetch handler depending on presence of alternate tags: + # set fetch handler depending on presence of alternate (or tuple) tags: self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): From a5ab4406d8b7d34da8e653d8d5debd2c0735b484 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sun, 21 Jun 2020 18:24:09 +0200 Subject: [PATCH 035/185] Removed unnecessary escape sequence This commit also contains changes to match requests that are 100% empty (by using "*" instead of "+" in the regex) --- config/filter.d/nginx-bad-request.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index 03721f86..2b8f5ab6 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -4,7 +4,8 @@ [Definition] # The request often doesn't contain a method, only some encoded garbage -failregex = ^ \- \S+ \[\] \"[^\"]+\" 400 +# This will also match requests that are entirely empty +failregex = ^ - \S+ \[\] "[^"]*" 400 datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? ^[^\[]*\[({DATE}) From 56fefe9240a6cc06650b91743339451caf6aa2f1 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sun, 21 Jun 2020 18:25:27 +0200 Subject: [PATCH 036/185] Added test file "nginx-bad-request" I tested with `./fail2ban-testcases testSampleRegex`, which did not return any errors. --- fail2ban/tests/files/logs/nginx-bad-request | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 fail2ban/tests/files/logs/nginx-bad-request diff --git a/fail2ban/tests/files/logs/nginx-bad-request b/fail2ban/tests/files/logs/nginx-bad-request new file mode 100644 index 00000000..a9ff6497 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-bad-request @@ -0,0 +1,23 @@ +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - root [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "\x03\x00\x00/*\xE0\x00\x00\x00\x00\x00Cookie: mstshash=Administr" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "GET //admin/pma/scripts/setup.php HTTP/1.1" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:54:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:54:28 +0100] "HELP" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:55:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:55:28 +0100] "batman" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T01:17:07", "match": true , "host": "7.8.9.10" } +7.8.9.10 - root [20/Jan/2015:01:17:07 +0100] "CONNECT 123.123.123.123 HTTP/1.1" 400 162 "-" "-" "-" + +# failJSON: { "time": "2014-12-12T22:59:02", "match": true , "host": "2.5.2.5" } +2.5.2.5 - tomcat [12/Dec/2014:22:59:02 +0100] "GET /cgi-bin/tools/tools.pl HTTP/1.1" 400 162 "-" "-" "-" \ No newline at end of file From 309c8dddd7adc2de140ed5a72088cd4f2dcc9b91 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Jun 2020 19:20:36 +0200 Subject: [PATCH 037/185] action.d/nftables.conf (type=multiport only): fixed port range selector (replacing `:` with `-`) --- config/action.d/nftables.conf | 2 +- fail2ban/tests/servertestcase.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf index c1fb8550..77cf3661 100644 --- a/config/action.d/nftables.conf +++ b/config/action.d/nftables.conf @@ -34,7 +34,7 @@ type = multiport rule_match-custom = rule_match-allports = meta l4proto \{ \} -rule_match-multiport = $proto dport \{ \} +rule_match-multiport = $proto dport \{ $(echo '' | sed s/:/-/g) \} match = > # Option: rule_stat diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index b771ab50..f1b667b1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1296,11 +1296,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), 'ip4-start': ( r"`nft add set inet f2b-table addr-set-j-w-nft-mp \{ type ipv4_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip saddr @addr-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip saddr @addr-set-j-w-nft-mp reject`", ), 'ip6-start': ( r"`nft add set inet f2b-table addr6-set-j-w-nft-mp \{ type ipv6_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip6 saddr @addr6-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip6 saddr @addr6-set-j-w-nft-mp reject`", ), 'flush': ( "`{ nft flush set inet f2b-table addr-set-j-w-nft-mp 2> /dev/null; } || ", From fd25c4cbb813db2562e5a2e54d4510eac291a897 Mon Sep 17 00:00:00 2001 From: TorontoMedia <10255876+TorontoMedia@users.noreply.github.com> Date: Sun, 28 Jun 2020 12:58:41 -0400 Subject: [PATCH 038/185] Remove duplicate method and rename invalid parameter --- fail2ban/server/failmanager.py | 4 ---- fail2ban/server/observer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index bc09c0e2..97b4c7a4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c19549ba..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: From 08dbe4abd593c09be8be9101964c2ee1915374f5 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 3 Jul 2020 13:45:29 +0200 Subject: [PATCH 039/185] fixed comment for loglevel, default is INFO --- config/fail2ban.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index ba0e9204..f3867839 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -19,7 +19,7 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # loglevel = INFO From ea35f2ad754810c51f5cf01d5fa3018002917a50 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 3 Jul 2020 13:47:46 +0200 Subject: [PATCH 040/185] default loglevel is INFO --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 662b3f48..4d01b6a1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -130,7 +130,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com The items that can be set in section [Definition] are: .TP .B loglevel -verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) +verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: INFO (equal 20) .TP .B logtarget log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local From 73a8175bb00ed2f456448f6edcfbf78ab7f0630e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 13:22:02 +0200 Subject: [PATCH 041/185] resolves names conflict (command action timeout and ipset timeout); closes gh-2790 --- config/action.d/firewallcmd-ipset.conf | 14 +++++++------- .../action.d/iptables-ipset-proto6-allports.conf | 14 +++++++------- config/action.d/iptables-ipset-proto6.conf | 14 +++++++------- config/action.d/shorewall-ipset-proto6.conf | 14 +++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 9dd9fbb2..42513933 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist actionunban = ipset del -exist @@ -40,18 +40,18 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index 4f200db0..addb2b95 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,18 +61,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 8956ec6a..7677564f 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,18 +61,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index cbcc5524..75eef218 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -76,16 +76,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) From 0ef8f6675d7c7d68e0b19d42071e1d1838f94627 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:25:31 +0200 Subject: [PATCH 042/185] fix travis builds (pipy in xenial, don't error if doc missing in default path after install) --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59a50313..064b678b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ matrix: - python: 2.7 name: 2.7 (xenial) - python: pypy - dist: trusty - python: 3.3 dist: trusty - python: 3.4 @@ -70,8 +69,8 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) - - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi + # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) + - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' # Test initd script - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: From 9510346507e70da6b25f27a20b3b6f6a84e8eeeb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:31:11 +0200 Subject: [PATCH 043/185] typo in skip message --- fail2ban/tests/fail2banregextestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..2da0098e 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -485,7 +485,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testLogtypeSystemdJournal(self): # pragma: no cover if not fail2banregex.FilterSystemd: - raise unittest.SkipTest('Skip test because no systemd backand available') + raise unittest.SkipTest('Skip test because no systemd backend available') self.assertTrue(_test_exec( "systemd-journal", FILTER_ZZZ_GEN +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' From 253d47d33c40c8f8f52b5d5fced6f2be7d942524 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:52:15 +0200 Subject: [PATCH 044/185] compat: some 2.x pypy versions produce UnicodeEncodeError: 'ascii' codec can't encode character on surrogates (uni_string must be fixed also for UTF-8 system encoding) --- fail2ban/tests/misctestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c1a6a345..458e9a23 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -201,7 +201,8 @@ class TestsUtilsTest(LogCaptureTestCase): uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') - uni_string(u'test\xcf') + if sys.version_info < (3,) and 'PyPy' not in sys.version: + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) From 98983adf761282ba4e6ad4d83a77fffcc6030573 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 17:14:13 +0200 Subject: [PATCH 045/185] update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 266ca614..65568f62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,8 @@ ver. 0.11.2-dev (20??/??/??) - development edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `` with all value of `` tags (gh-2755) ver. 0.11.1 (2020/01/11) - this-is-the-way From 2216fd8da4e95564bc4cd0047ffae08d24d3d17d Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Tue, 4 Aug 2020 19:04:05 -0400 Subject: [PATCH 046/185] Add Apprise Support (50+ Notifications) --- MANIFEST | 1 + config/action.d/apprise.conf | 47 ++++++++++++++++++++++++++++++++++++ config/jail.conf | 9 +++++++ 3 files changed, 57 insertions(+) create mode 100644 config/action.d/apprise.conf diff --git a/MANIFEST b/MANIFEST index ed441bac..d02b8bd0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,6 +3,7 @@ bin/fail2ban-regex bin/fail2ban-server bin/fail2ban-testcases ChangeLog +config/action.d/apprise.conf config/action.d/abuseipdb.conf config/action.d/apf.conf config/action.d/badips.conf diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf new file mode 100644 index 00000000..ac54d8fd --- /dev/null +++ b/config/action.d/apprise.conf @@ -0,0 +1,47 @@ +# Fail2Ban configuration file +# +# Author: Chris Caron +# +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = printf %%b "The jail as been started successfully." |apprise -t "[Fail2Ban] : started on `uname -n`" -c /etc/fail2ban/apprise.conf + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = printf %%b "The jail has been stopped." |apprise -t "[Fail2Ban] : stopped on `uname -n`" -c /etc/fail2ban/apprise.conf + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | apprise -n "warning" -t "[Fail2Ban] : banned $ + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] + +# Define location of the default apprise configuration file to use +# +config = /etc/fail2ban/apprise.conf diff --git a/config/jail.conf b/config/jail.conf index 6e8a6a2f..dbca2e8f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -227,6 +227,15 @@ action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(proto action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[name=%(__name__)s, config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +action_apprise = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + apprise[name=%(__name__)s] + # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] From a7ad3e00ddc3a3a71e8ab5a20dcfb33f63daa0e2 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 11:54:54 +0200 Subject: [PATCH 047/185] amend to 91eca4fdeb728fa0fd2d6cf55db8eb7baa2a39a2 (#2634): server creates a RTM-directory for socket/pid file automatically (don't check its existence in client) --- fail2ban/client/fail2banclient.py | 13 ------------- fail2ban/tests/fail2banclienttestcase.py | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 7c90ca40..6ea18fda 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -168,19 +168,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): if not ret: return None - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self._conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self._conf["socket"])) - return None - if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return None - # Check already running if not self._conf["force"] and os.path.exists(self._conf["socket"]): logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index a334b568..fd6d6bbd 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -623,12 +623,6 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start") - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## not running self.execCmd(FAILED, (), "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload") @@ -724,12 +718,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock")) - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## already exists: open(pjoin(tmp, "f2b.sock"), 'a').close() self.execCmd(FAILED, (), From 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 13:57:36 +0200 Subject: [PATCH 048/185] closes gh-2758: no explicit flush (close std-channels on exit, it would cause implicit flush without to produce an error 32 "Broken pipe" on closed pipe) --- fail2ban/client/fail2bancmdline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 53c86de6..8936e03f 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -308,6 +308,10 @@ class Fail2banCmdLine(): # since method is also exposed in API via globally bound variable @staticmethod def _exit(code=0): + # implicit flush without to produce broken pipe error (32): + sys.stderr.close() + sys.stdout.close() + # exit: if hasattr(os, '_exit') and os._exit: os._exit(code) else: @@ -318,8 +322,6 @@ class Fail2banCmdLine(): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() # exit Fail2banCmdLine._exit(code) From 7d172faa50db8153265b7ac0e26f1034d7d6119f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 15:37:19 +0200 Subject: [PATCH 049/185] implements gh-2791: fail2ban-client extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS) --- ChangeLog | 1 + fail2ban/server/actions.py | 14 +++++++++++++- fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index fb38aca1..9b30bd6a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 6123605d..4689b9d7 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -251,7 +251,7 @@ class Actions(JailThread, Mapping): if ip is None: return self.__flushBan(db) # Multiple IPs: - if isinstance(ip, list): + if isinstance(ip, (list, tuple)): missed = [] cnt = 0 for i in ip: @@ -273,6 +273,18 @@ class Actions(JailThread, Mapping): # Unban the IP. self.__unBan(ticket) else: + # Multiple IPs by subnet or dns: + if not isinstance(ip, IPAddr): + ipa = IPAddr(ip) + if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): + ips = filter( + lambda i: ( + isinstance(i, IPAddr) and (i == ipa or i.isSingle and i.isInNet(ipa)) + ), self.__banManager.getBanList() + ) + if ips: + return self.removeBannedIP(ips, db, ifexists) + # not found: msg = "%s is not banned" % ip logSys.log(logging.MSG, msg) if ifexists: diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index fd6d6bbd..104e4c57 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1158,13 +1158,26 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertNotLogged("[test-jail1] Found 192.0.2.5") # unban single ips: - self.pruneLog("[test-phase 6]") + self.pruneLog("[test-phase 6a]") self.execCmd(SUCCESS, startparams, "--async", "unban", "192.0.2.5", "192.0.2.6") self.assertLogged( "192.0.2.5 is not banned", "[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME ) + # unban ips by subnet (cidr/mask): + self.pruneLog("[test-phase 6b]") + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.2/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.2", + "[test-jail1] Unban 192.0.2.3", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.8/31", "192.0.2.100/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.8", + "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) # reload all (one jail) with unban all: self.pruneLog("[test-phase 7]") @@ -1175,8 +1188,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Jail 'test-jail1' reloaded", "[test-jail1] Unban 192.0.2.1", - "[test-jail1] Unban 192.0.2.2", - "[test-jail1] Unban 192.0.2.3", "[test-jail1] Unban 192.0.2.4", all=True ) # no restart occurred, no more ban (unbanned all using option "--unban"): @@ -1184,8 +1195,6 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", "[test-jail1] Ban 192.0.2.1", - "[test-jail1] Ban 192.0.2.2", - "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", all=True ) From 3ca69c8c0a756b58c751707e72905d3f50c4a620 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 17:14:21 +0200 Subject: [PATCH 050/185] amend to #2791: unban subnet when subnet is in supplied subnet --- fail2ban/server/actions.py | 6 +----- fail2ban/server/ipdns.py | 5 +++++ fail2ban/tests/fail2banclienttestcase.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 4689b9d7..3308d4b2 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -277,11 +277,7 @@ class Actions(JailThread, Mapping): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter( - lambda i: ( - isinstance(i, IPAddr) and (i == ipa or i.isSingle and i.isInNet(ipa)) - ), self.__banManager.getBanList() - ) + ips = filter(ipa.contains, self.__banManager.getBanList()) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..571ccc4f 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -517,6 +517,11 @@ class IPAddr(object): return (self.addr & mask) == net.addr + def contains(self, ip): + """Return whether the object (as network) contains given IP + """ + return isinstance(ip, IPAddr) and (ip == self or ip.isInNet(self)) + # Pre-calculated map: addr to maskplen def __getMaskMap(): m6 = (1 << 128)-1 diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 104e4c57..bbd6964a 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1179,6 +1179,21 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail1] Unban 192.0.2.8", "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) + # ban/unban subnet(s): + self.pruneLog("[test-phase 6c]") + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "banip", "192.0.2.96/28", "192.0.2.112/28") + self.assertLogged( + "[test-jail1] Ban 192.0.2.96/28", + "[test-jail1] Ban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "unbanip", "192.0.2.64/26"); # contains both subnets .96/28 and .112/28 + self.assertLogged( + "[test-jail1] Unban 192.0.2.96/28", + "[test-jail1] Unban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + # reload all (one jail) with unban all: self.pruneLog("[test-phase 7]") self.execCmd(SUCCESS, startparams, From 7e8d98c4edf7ecd5b1805eb850225a0a8435320d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Aug 2020 19:20:27 +0200 Subject: [PATCH 051/185] code review, fix simplest TZ issue - avoid date adjustment by assuming of last year (date without year in the future) by wrong zone (don't adjust by offset up to +24 hours) --- fail2ban/server/filter.py | 7 +++++-- fail2ban/server/filterpoll.py | 4 +++- fail2ban/server/strptime.py | 5 ++--- fail2ban/tests/files/logs/courier-smtp | 6 +++--- fail2ban/tests/filtertestcase.py | 8 +++++--- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a92acb8b..1ff81d65 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -633,6 +633,9 @@ class Filter(JailThread): fail = element[3] logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) + # ensure the time is not in the future, e. g. by some estimated (assumed) time: + if self.checkFindTime and unixTime > MyTime.time(): + unixTime = MyTime.time() tick = FailTicket(ip, unixTime, data=fail) if self._inIgnoreIPList(ip, tick): continue @@ -936,7 +939,7 @@ class FileFilter(Filter): log.setPos(lastpos) self.__logs[path] = log logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) - if autoSeek: + if autoSeek and not tail: self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific @@ -1206,7 +1209,7 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail = False): + def __init__(self, filename, encoding, tail=False): self.__filename = filename self.setEncoding(encoding) self.__tail = tail diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index b4d8ab14..7bbdfc5c 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -111,6 +111,8 @@ class FilterPoll(FileFilter): modlst = [] Utils.wait_for(lambda: not self.active or self.getModified(modlst), self.sleeptime) + if not self.active: # pragma: no cover - timing + break for filename in modlst: self.getFailures(filename) self.__modified = True @@ -140,7 +142,7 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) stats = logStats.st_mtime, logStats.st_ino, logStats.st_size - pstats = self.__prevStats.get(filename, (0)) + pstats = self.__prevStats.get(filename, (0,)) if logSys.getEffectiveLevel() <= 4: # we do not want to waste time on strftime etc if not necessary dt = logStats.st_mtime - pstats[0] diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 498d284b..1464a96d 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -291,9 +291,8 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): date_result -= datetime.timedelta(days=1) if assume_year: if not now: now = MyTime.now() - if date_result > now: - # Could be last year? - # also reset month and day as it's not yesterday... + if date_result > now + datetime.timedelta(days=1): # ignore by timezone issues (+24h) + # assume last year - also reset month and day as it's not yesterday... date_result = date_result.replace( year=year-1, month=month, day=day) diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index cea73073..6da0d0a4 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -8,9 +8,9 @@ Jul 4 18:39:39 mail courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin -# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +# failJSON: { "time": "2005-08-14T12:51:05", "match": true , "host": "192.0.2.3" } Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index a511b5d0..e8634929 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1606,7 +1606,7 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 7, 1124013544.0) + output = ('203.162.223.135', 9, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1615,7 +1615,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... - output = ('203.162.223.135', 5, 1124013544.0) + output = ('203.162.223.135', 3, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1624,7 +1624,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... - output = ('203.162.223.135', 1, 1124013544.0) + output = ('203.162.223.135', 2, 1124013600.0) self.filter.setMaxRetry(1) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) @@ -1652,6 +1652,7 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresWrongChar(self): + self.filter.checkFindTime = False # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') @@ -1672,6 +1673,7 @@ class GetFailures(LogCaptureTestCase): for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') From d2cef96f33d13acad7d3dd1129de44c0f723efd6 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 18:52:00 +0200 Subject: [PATCH 052/185] filter: implement mode `inOperation`, which gets activated if filter starts processing of new messages; better interaction with non-matching optional datepattern or invalid timestamps (or timezone) - assuming now instead of bypass; fixed test cases gathering new failures now in operation mode --- fail2ban/server/filter.py | 118 ++++++++++++------ fail2ban/server/utils.py | 4 + fail2ban/tests/fail2banregextestcase.py | 6 +- fail2ban/tests/files/logs/zzz-generic-example | 8 +- fail2ban/tests/filtertestcase.py | 44 ++++--- 5 files changed, 118 insertions(+), 62 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 1ff81d65..40bfeef5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -112,6 +112,8 @@ class Filter(JailThread): self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## shows that filter is in operation mode (processing new messages): + self.inOperation = True ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): self.banASAP = True ## Ticks counter @@ -586,16 +588,26 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, False) return False + def _logWarnOnce(self, nextLTM, *args): + """Log some issue as warning once per day, otherwise level 7""" + if MyTime.time() < getattr(self, nextLTM, 0): + if logSys.getEffectiveLevel() <= 7: logSys.log(7, *(args[0])) + else: + setattr(self, nextLTM, MyTime.time() + 24*60*60) + for args in args: + logSys.warning('[%s] ' + args[0], self.jailName, *args[1:]) + def processLine(self, line, date=None): """Split the time portion from log msg and return findFailures on them """ + logSys.log(7, "Working on line %r", line) + + noDate = False if date: tupleLine = line self.__lastTimeText = tupleLine[1] self.__lastDate = date else: - logSys.log(7, "Working on line %r", line) - # try to parse date: timeMatch = self.dateDetector.matchTime(line) m = timeMatch[0] @@ -606,22 +618,51 @@ class Filter(JailThread): tupleLine = (line[:s], m, line[e:]) if m: # found and not empty - retrive date: date = self.dateDetector.getTime(m, timeMatch) - - if date is None: - if m: logSys.error("findFailure failed to parse timeText: %s", m) + if date is not None: + # Lets get the time part + date = date[0] + self.__lastTimeText = m + self.__lastDate = date + else: + logSys.error("findFailure failed to parse timeText: %s", m) + else: + tupleLine = ("", "", line) + # still no date - try to use last known: + if date is None: + noDate = True + if self.__lastDate and self.__lastDate > MyTime.time() - 60: + tupleLine = ("", self.__lastTimeText, line) date = self.__lastDate - else: - # Lets get the time part - date = date[0] - self.__lastTimeText = m + + if self.checkFindTime: + # if in operation (modifications have been really found): + if self.inOperation: + # if weird date - we'd simulate now for timeing issue (too large deviation from now): + if (date is None or date < MyTime.time() - 60 or date > MyTime.time() + 60): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_simByTimeWarn", + ("Simulate NOW in operation since found time has too large deviation %s ~ %s +/- %s", + date, MyTime.time(), 60), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # simulate now as date: + date = MyTime.time() self.__lastDate = date else: - tupleLine = (line, self.__lastTimeText, "") - date = self.__lastDate + # in initialization (restore) phase, if too old - ignore: + if date is not None and date < MyTime.time() - self.getFindTime(): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_ignByTimeWarn", + ("Ignore line since time %s < %s - %s", + date, MyTime.time(), self.getFindTime()), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # ignore - too old (obsolete) entry: + return [] # save last line (lazy convert of process line tuple to string on demand): self.processedLine = lambda: "".join(tupleLine[::2]) - return self.findFailure(tupleLine, date) + return self.findFailure(tupleLine, date, noDate=noDate) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager @@ -755,7 +796,7 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date): + def findFailure(self, tupleLine, date, noDate=False): failList = list() ll = logSys.getEffectiveLevel() @@ -765,11 +806,6 @@ class Filter(JailThread): returnRawHost = True cidr = IPAddr.CIDR_RAW - if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): - if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", - date, MyTime.time(), self.getFindTime()) - return failList - if self.__lineBufferSize > 1: self.__lineBuffer.append(tupleLine) orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:] @@ -820,17 +856,17 @@ class Filter(JailThread): if not self.checkAllRegex: break continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " + if noDate: + self._logWarnOnce("_next_noTimeWarn", + ("Found a match but no valid date/time found for %r.", tupleLine[1]), + ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), + ("Please try setting a custom " "date pattern (see man page jail.conf(5)). " "If format is complex, please " "file a detailed issue on" " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), tupleLine[1]) - continue + "in order to get support for this format.",)) + if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): if not self.checkAllRegex or self.__lineBufferSize > 1: self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None @@ -1023,7 +1059,7 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename): + def getFailures(self, filename, inOperation=None): log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) @@ -1068,9 +1104,14 @@ class FileFilter(Filter): if has_content: while not self.idle: line = log.readline() - if not line or not self.active: - # The jail reached the bottom or has been stopped + if not self.active: break; # jail has been stopped + if not line: + # The jail reached the bottom, simply set in operation for this log + # (since we are first time at end of file, growing is only possible after modifications): + log.inOperation = True break + # acquire in operation from log and process: + self.inOperation = inOperation if inOperation is not None else log.inOperation self.processLineAndAdd(line.rstrip('\r\n')) finally: log.close() @@ -1230,6 +1271,8 @@ class FileContainer: self.__pos = 0 finally: handler.close() + ## shows that log is in operation mode (expecting new messages only from here): + self.inOperation = tail def getFileName(self): return self.__filename @@ -1303,16 +1346,17 @@ class FileContainer: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: global _decode_line_warn - lev = logging.DEBUG - if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = 7 + if not _decode_line_warn.get(filename, 0): lev = logging.WARNING - _decode_line_warn[filename] = MyTime.time() + 24*60*60 + _decode_line_warn.set(filename, 1) logSys.log(lev, - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", - filename, enc, line) + "Error decoding line from '%s' with '%s'.", filename, enc) + if logSys.getEffectiveLevel() <= lev: + logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r", + line) # decode with replacing error chars: line = line.decode(enc, 'replace') return line @@ -1333,7 +1377,7 @@ class FileContainer: ## print "D: Closed %s with pos %d" % (handler, self.__pos) ## sys.stdout.flush() -_decode_line_warn = {} +_decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); ## diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 053aa04f..4e64ca0b 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -125,6 +125,10 @@ class Utils(): with self.__lock: self._cache.pop(k, None) + def clear(self): + with self.__lock: + self._cache.clear() + @staticmethod def setFBlockMode(fhandle, value): diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 2da0098e..2e1d18db 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -172,7 +172,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') self.assertLogged('Error decoding line'); self.assertLogged('Continuing to process line ignoring invalid characters') @@ -186,7 +186,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 19 matched, 0 missed') def testDirectRE_1raw_noDns(self): self.assertTrue(_test_exec( @@ -194,7 +194,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", "--usedns=no", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') # usage of \S+ causes raw handling automatically: self.pruneLog() self.assertTrue(_test_exec( diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index d0bd3322..118c7e12 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } +# -- wrong time direct in journal-line (used last known date or now, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } +# -- wrong time direct in journal-line (used last known date, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index e8634929..63f43b21 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -215,7 +215,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) - fout.write('\n'.join(lines)) + fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases # Opened earlier, therefore must close it @@ -878,7 +878,7 @@ class LogFileMonitor(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -897,7 +897,7 @@ class LogFileMonitor(LogCaptureTestCase): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.filter.getFailures(self.name) #self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -916,7 +916,7 @@ class LogFileMonitor(LogCaptureTestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1027,13 +1027,13 @@ def get_monitor_failures_testcase(Filter_): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=12) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and our dummy jail is empty as well self.assertFalse(len(self.jail)) # since it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) if idle: self.waitForTicks(1) self.assertTrue(self.isEmpty(1)) @@ -1052,7 +1052,7 @@ def get_monitor_failures_testcase(Filter_): #return # just for fun let's copy all of them again and see if that results # in a new ban - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.assert_correct_last_attempt(GetFailures.FAILURES_01) def test_rewrite_file(self): @@ -1066,7 +1066,7 @@ def get_monitor_failures_testcase(Filter_): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) def _wait4failures(self, count=2): @@ -1087,13 +1087,13 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1169,8 +1169,7 @@ def get_monitor_failures_testcase(Filter_): def _test_move_into_file(self, interim_kill=False): # if we move a new file into the location of an old (monitored) file - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() # make sure that it is monitored first self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1181,14 +1180,14 @@ def get_monitor_failures_testcase(Filter_): # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', - n=100).close() + skip=12, n=3).close() os.rename(self.name + '.new', self.name) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) # and to make sure that it now monitored for changes _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 9) @@ -1207,7 +1206,7 @@ def get_monitor_failures_testcase(Filter_): # create a bogus file in the same directory and see if that doesn't affect open(self.name + '.bak2', 'w').close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') @@ -1239,8 +1238,8 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above # now copy and get even more - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) - # check for 3 failures (not 9), because 6 already get above... + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) + # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: self.assertEqual(self.filter.failManager.getFailTotal(), 12) @@ -1606,13 +1605,21 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 9, 1124013600.0) + output = ('203.162.223.135', 6, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_InOperation(self): + output = ('203.162.223.135', 9, 1124013600.0) + + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) + _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... output = ('203.162.223.135', 3, 1124013600.0) @@ -1673,6 +1680,7 @@ class GetFailures(LogCaptureTestCase): for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + if DefLogSys.getEffectiveLevel() > 7: DefLogSys.setLevel(7); # ensure decode_line logs always self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: From b82f584a96fd3a19b157fc85c51a72c952809713 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 19:33:40 +0200 Subject: [PATCH 053/185] added test case covering new date handling (simulation, unknown format, warnings, etc) --- fail2ban/server/filter.py | 9 +++---- fail2ban/tests/filtertestcase.py | 44 +++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 40bfeef5..3ea0a601 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -651,6 +651,7 @@ class Filter(JailThread): else: # in initialization (restore) phase, if too old - ignore: if date is not None and date < MyTime.time() - self.getFindTime(): + print('**********') # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", ("Ignore line since time %s < %s - %s", @@ -860,12 +861,8 @@ class Filter(JailThread): self._logWarnOnce("_next_noTimeWarn", ("Found a match but no valid date/time found for %r.", tupleLine[1]), ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), - ("Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.",)) + ("Please try setting a custom date pattern (see man page jail.conf(5)).",) + ) if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): if not self.checkAllRegex or self.__lineBufferSize > 1: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 63f43b21..2dac91d1 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -394,12 +394,13 @@ class IgnoreIP(LogCaptureTestCase): finally: tearDownMyTime() - def testTimeJump(self): + def _testTimeJump(self, inOperation=False): try: self.filter.addFailRegex('^') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') self.filter.setFindTime(10); # max 10 seconds back self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = inOperation # self.pruneLog('[phase 1] DST time jump') # check local time jump (DST hole): @@ -430,6 +431,47 @@ class IgnoreIP(LogCaptureTestCase): self.assertNotLogged('Ignore line') finally: tearDownMyTime() + def testTimeJump(self): + self._testTimeJump(inOperation=False) + def testTimeJump_InOperation(self): + self._testTimeJump(inOperation=True) + + def testWrongTimeZone(self): + try: + self.filter.addFailRegex('fail from $') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = True; # real processing (all messages are new) + # current time is 1h later than log-entries: + MyTime.setTime(1572138000+3600) + # + self.pruneLog("[phase 1] simulate wrong TZ") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 + self.assertLogged( + "Simulate NOW in operation since found time has too large deviation", + "Please check jail has possibly a timezone issue.", + "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", + "Total # of detected failures: 3.", wait=True) + # + self.pruneLog("[phase 2] wrong TZ given in log") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 + self.assertLogged( + "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", + "Total # of detected failures: 6.", all=True, wait=True) + self.assertNotLogged("Found a match but no valid date/time found") + # + self.pruneLog("[phase 3] other timestamp (don't match datepattern), regex matches") + for i in range(3): + self.filter.processLineAndAdd('27.10.2019 04:00:00 fail from 192.0.2.17'); # +3 = 9 + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "192.0.2.17:1", "192.0.2.17:2", "192.0.2.17:3", + "Total # of detected failures: 9.", all=True, wait=True) + finally: + tearDownMyTime() def testAddAttempt(self): self.filter.setMaxRetry(3) From f21c58dc7249fca1c35b50d5cd6d31bbb0fdcd42 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 20:27:39 +0200 Subject: [PATCH 054/185] implements special datepattern `{NONE}` - allow to find failures without date-time in log messages (filter use now as timestamp) closes gh-2802 --- fail2ban/server/datedetector.py | 2 ++ fail2ban/server/filter.py | 6 +++++- fail2ban/tests/fail2banregextestcase.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 0a6451be..90a70b0d 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -282,6 +282,8 @@ class DateDetector(object): elif "{DATE}" in key: self.addDefaultTemplate(preMatch=pattern, allDefaults=False) return + elif key == "{NONE}": + template = _getPatternTemplate('{UNB}^', key) else: template = _getPatternTemplate(pattern, key) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 3ea0a601..d10da7ab 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -625,6 +625,11 @@ class Filter(JailThread): self.__lastDate = date else: logSys.error("findFailure failed to parse timeText: %s", m) + else: + # matched empty value - date is optional or not available - set it to now: + date = MyTime.time() + self.__lastTimeText = "" + self.__lastDate = date else: tupleLine = ("", "", line) # still no date - try to use last known: @@ -651,7 +656,6 @@ class Filter(JailThread): else: # in initialization (restore) phase, if too old - ignore: if date is not None and date < MyTime.time() - self.getFindTime(): - print('**********') # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", ("Ignore line since time %s < %s - %s", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 2e1d18db..4d878a24 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -81,6 +81,7 @@ def _test_exec_command_line(*args): return _exit_code STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" +STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " RE_00_ID = r"Authentication failure for .*? from $" @@ -361,6 +362,24 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('192.0.2.0, kevin, inet4') self.pruneLog() + def testNoDateTime(self): + # datepattern doesn't match: + self.assertTrue(_test_exec('-d', '{^LN-BEG}EPOCH', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "Found-ID:kevin", all=True) + self.pruneLog() + # explicitly no datepattern: + self.assertTrue(_test_exec('-d', '{NONE}', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found-ID:kevin", all=True) + self.assertNotLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", all=True) + + self.pruneLog() + def testFrmtOutputWrapML(self): unittest.F2B.SkipIfCfgMissing(stock=True) # complex substitution using tags and message (ip, user, msg): From 76e5d2b1998265d33f7ae1b30bb72b02d0e1515f Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 21 Aug 2020 17:11:56 +0200 Subject: [PATCH 055/185] amend to f21c58dc7249fca1c35b50d5cd6d31bbb0fdcd42, better follow previous handling with last known datetime (compatibility for multi-line logs, in case of second line without a timestamp) --- fail2ban/server/filter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d10da7ab..4e947d27 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -625,11 +625,14 @@ class Filter(JailThread): self.__lastDate = date else: logSys.error("findFailure failed to parse timeText: %s", m) + # matched empty value - date is optional or not available - set it to last known or now: + elif self.__lastDate and self.__lastDate > MyTime.time() - 60: + # set it to last known: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate else: - # matched empty value - date is optional or not available - set it to now: + # set it to now: date = MyTime.time() - self.__lastTimeText = "" - self.__lastDate = date else: tupleLine = ("", "", line) # still no date - try to use last known: From 295630cccfb8409a4e590414d968487eb8596ccb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Aug 2020 16:12:55 +0200 Subject: [PATCH 056/185] documentation and changelog --- ChangeLog | 13 ++++++++++--- man/jail.conf.5 | 22 +++++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9b30bd6a..59624d42 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,12 +59,19 @@ ver. 0.10.6-dev (20??/??/??) - development edition - `aggressive`: matches 401 and any variant (with and without username) * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) -### New Features +### New Features and Enhancements * new filter and jail for GitLab recognizing failed application logins (gh-2689) - -### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; +* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) + as well as some warnings signaling user about invalid pattern or zone (gh-2814): + - filter gets mode in-operation, which gets activated if filter starts processing of new messages; + in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much + from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected + bypass of failure (previously exceeding `findtime`); + - better interaction with non-matching optional datepattern or invalid timestamps; + - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages, + whereas filter will use now as timestamp (gh-2802) * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); * fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 4d01b6a1..830c8aed 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -460,11 +460,27 @@ Similar to actions, filters have an [Init] section which can be overridden in \f specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .TP \fBdatepattern\fR -specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %Y-%m-%d %H:%M(?::%S)?. For a list of valid format directives, see Python library documentation for strptime behaviour. -.br -Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used. +specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %%Y-%%m-%%d %%H:%%M(?::%%S)?. +For a list of valid format directives, see Python library documentation for strptime behaviour. .br \fBNOTE:\fR due to config file string substitution, that %'s must be escaped by an % in config files. +.br +Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used as datepattern. +.br +Normally the regexp generated for datepattern additionally gets word-start and word-end boundaries to avoid accidental match inside of some word in a message. +There are several prefixes and words with special meaning that could be specified with custom datepattern to control resulting regex: +.RS +.IP +\fI{DEFAULT}\fR - can be used to add default date patterns of fail2ban. +.IP +\fI{DATE}\fR - can be used as part of regex that will be replaced with default date patterns. +.IP +\fI{^LN-BEG}\fR - prefix (similar to \fI^\fR) changing word-start boundary to line-start boundary (ignoring up to 2 characters). If used as value (not as a prefix), it will also set all default date patterns (similar to \fI{DEFAULT}\fR), but anchored at begin of message line. +.IP +\fI{UNB}\fR - prefix to disable automatic word boundaries in regex. +.IP +\fI{NONE}\fR - value would allow to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). +.RE .TP \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. From ad51fb7e1ea42c795135fe97970c31cfce51e446 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Aug 2020 16:41:22 +0200 Subject: [PATCH 057/185] partial cherry-pick fd25c4cbb813db2562e5a2e54d4510eac291a897 (#2768) --- fail2ban/server/failmanager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index eee979fd..3458aed5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value From 1e3da21c680ac9a282f72348869e8cdb3d1e0399 Mon Sep 17 00:00:00 2001 From: TorontoMedia <10255876+TorontoMedia@users.noreply.github.com> Date: Sun, 28 Jun 2020 12:58:41 -0400 Subject: [PATCH 058/185] Remove duplicate method and rename invalid parameter (cherry picked from commit fd25c4cbb813db2562e5a2e54d4510eac291a897) --- fail2ban/server/failmanager.py | 4 ---- fail2ban/server/observer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index bc09c0e2..97b4c7a4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c19549ba..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: From 1707560df8033341e4fb8a2e79db3ea684b42386 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Wed, 26 Feb 2020 10:41:55 +0100 Subject: [PATCH 059/185] Enhance Guacamole jail --- config/filter.d/guacamole.conf | 50 ++++++++++++++++++++++------- config/jail.conf | 1 + fail2ban/tests/files/logs/guacamole | 5 +++ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 09b4e7b0..bc6dbea9 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -5,21 +5,47 @@ [Definition] -# Option: failregex -# Notes.: regex to match the password failures messages in the logfile. -# Values: TEXT -# +logging = catalina +failregex = /failregex> +maxlines = /maxlines> +datepattern = /datepattern> + +[L_catalina] + failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" failed\.$ -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# -ignoreregex = - -# "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p ^WARNING:()** - {^LN-BEG} \ No newline at end of file + {^LN-BEG} + +[L_webapp] + +failregex = ^ \[\S+\] WARN \S+ - Authentication attempt from for user "[^"]+" failed. + +maxlines = 1 + +datepattern = ^%%H:%%M:%%S.%%f + +# DEV Notes: +# +# failregex is based on the default pattern given in Guacamole documentation : +# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging +# +# The following logback.xml Guacamole configuration file can then be used accordingly : +# +# +# /var/log/guacamole.log +# +# /var/log/guacamole.%d.log.gz +# 32 +# +# +# %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n +# +# +# +# +# +# diff --git a/config/jail.conf b/config/jail.conf index 39ebbe6d..40827707 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -440,6 +440,7 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole index 3de67454..ebb7afb0 100644 --- a/fail2ban/tests/files/logs/guacamole +++ b/fail2ban/tests/files/logs/guacamole @@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" } WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. + +# filterOptions: {"logging": "webapp"} + +# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" } +12:57:32.907 [http-nio-8080-exec-10] WARN o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed. From 7b05c1ce7a487f2c269815f8a17b7444213ae555 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 14:52:22 +0200 Subject: [PATCH 060/185] do type-convert only in getCombined (otherwise int/bool conversion prevents substitution or section-related interpolation of tags) --- fail2ban/client/actionreader.py | 11 ++---- fail2ban/client/configreader.py | 54 ++++++++++++++++++++------ fail2ban/tests/clientreadertestcase.py | 7 ++-- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 131e37cb..011a213d 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,17 +38,17 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], - "actionrepair_on_unban": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): @@ -83,11 +83,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 20709b72..f9a6d288 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -228,7 +228,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} @@ -244,12 +244,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue + if convert: + if opttype == "bool": + v = self.getboolean(sec, optname) + if v is None: continue + elif opttype == "int": + v = self.getint(sec, optname) + if v is None: continue + else: + v = self.get(sec, optname, vars=pOptions) else: v = self.get(sec, optname, vars=pOptions) values[optname] = v @@ -267,7 +270,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue @@ -324,8 +327,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -346,10 +350,34 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, pOptions={}): + options = self._configOpts + for optname in options: + if isinstance(options, (list,tuple)): + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + else: + opttype, optvalue = options[optname] + if optname in pOptions: + continue + try: + if opttype == "bool": + v = opts.get(optname) + if v is None or isinstance(v, bool): continue + v = _as_bool(v) + opts[optname] = v + elif opttype == "int": + v = opts.get(optname) + if v is None or isinstance(v, (int, long)): continue + v = int(v) + opts[optname] = v + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -384,6 +412,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 2c1d0a0e..ae39c157 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -490,7 +490,9 @@ class FilterReaderTest(unittest.TestCase): self.__share_cfg = {} def testConvert(self): - output = [['multi-set', 'testcase01', 'addfailregex', [ + output = [ + ['set', 'testcase01', 'maxlines', 1], + ['multi-set', 'testcase01', 'addfailregex', [ "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" @@ -512,7 +514,6 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addjournalmatch', "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], ['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"], - ['set', 'testcase01', 'maxlines', 1], # Last for overide test ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) @@ -529,7 +530,7 @@ class FilterReaderTest(unittest.TestCase): filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - output[-1][-1] = "5" + output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) def testFilterReaderSubstitionDefault(self): From d9b8796792ff2d876fe02e5a469767d33469c276 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:01:34 +0200 Subject: [PATCH 061/185] amend with better (common) handling, documentation and tests --- fail2ban/client/configreader.py | 93 +++++++++++++------------- fail2ban/tests/clientreadertestcase.py | 30 +++++++-- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index f9a6d288..d8817bed 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -233,29 +257,17 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if convert: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue - else: - v = self.get(sec, optname, vars=pOptions) - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -350,33 +362,20 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) - def convertOptions(self, opts, pOptions={}): - options = self._configOpts - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] - if optname in pOptions: - continue - try: - if opttype == "bool": - v = opts.get(optname) - if v is None or isinstance(v, bool): continue - v = _as_bool(v) - opts[optname] = v - elif opttype == "int": - v = opts.get(optname) - if v is None or isinstance(v, (int, long)): continue - v = int(v) - opts[optname] = v - except ValueError: - logSys.warning("Wrong %s value %r for %r. Using default one: %r", - opttype, v, optname, optvalue) - opts[optname] = optvalue + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init @@ -413,7 +412,7 @@ class DefinitionInitConfigReader(ConfigReader): if not opts: raise ValueError('recursive tag definitions unable to be resolved') # convert options after all interpolations: - self.convertOptions(opts) + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ae39c157..8abfd4a5 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -87,6 +87,21 @@ option = %s self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + def testConvert(self): + self.c.add_section("Definition") + self.c.set("Definition", "a", "1") + self.c.set("Definition", "b", "1") + self.c.set("Definition", "c", "test") + opts = self.c.getOptions("Definition", + (('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + opts = self.c.getOptions("Definition", + (('int', 'a'), ('bool', 'b'), ('int', 'c'))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None}) + opts = self.c.getOptions("Definition", + {'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)}) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + def testInaccessibleFile(self): f = os.path.join(self.d, "d.conf") # inaccessible file self._write('d.conf', 0) @@ -483,11 +498,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertRaises(NoSectionError, c.getOptions, 'test', {}) -class FilterReaderTest(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(FilterReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} +class FilterReaderTest(LogCaptureTestCase): def testConvert(self): output = [ @@ -533,6 +544,15 @@ class FilterReaderTest(unittest.TestCase): output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) + def testConvertOptions(self): + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '', 'test': 'X'}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined(); + self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines' + self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:") + def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {}, From 2945fe8cbd39b6e02ae8065878ec2dffc68f0021 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:25:32 +0200 Subject: [PATCH 062/185] changelog --- ChangeLog | 3 +++ fail2ban/client/configreader.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59624d42..cacfdee7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* fixed type conversion in config readers (take place after all interpolations get ready), that allows to + specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters. * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing @@ -61,6 +63,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### New Features and Enhancements * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d8817bed..1b5a56a2 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -282,7 +282,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue From 5a2cc4e1c5219645ca801c8ba90bab5c609340ec Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:05:20 +0200 Subject: [PATCH 063/185] substituteRecursiveTags: more precise self- or cyclic-recursion prevention (don't clear replacement counts of tags, rather consider replacement count by tax X in tag Y) --- fail2ban/helpers.py | 14 ++++++++------ fail2ban/tests/actiontestcase.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 3ef7d543..dc7852ae 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -398,8 +398,8 @@ def splitWithOptions(option): # tags () in tagged options. # -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 +# max tag replacement count (considering tag X in tag Y repeat): +MAX_TAG_REPLACE_COUNT = 25 # compiled RE for tag name (replacement name) TAG_CRE = re.compile(r'<([^ <>]+)>') @@ -433,6 +433,7 @@ def substituteRecursiveTags(inptags, conditional='', done = set() noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) + repCounts = {} while True: repFlag = False # substitute each value: @@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='', value = orgval = uni_string(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) - refCounts = {} + rplc = repCounts.get(tag, {}) #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: # found replacement tag: @@ -454,13 +455,13 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue #logSys.log(5, 'found: %s' % rtag) - if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: + if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, rtag, refCounts, value)) + (tag, rtag, rplc, value)) repl = None if conditional: repl = tags.get(rtag + '?' + conditional) @@ -480,7 +481,7 @@ def substituteRecursiveTags(inptags, conditional='', value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[rtag] = refCounts.get(rtag, 0) + 1 + rplc[rtag] = rplc.get(rtag, 0) + 1 # the next match for replace: m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) @@ -488,6 +489,7 @@ def substituteRecursiveTags(inptags, conditional='', if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): if tre_search(value): + repCounts[tag] = rplc repFlag = True # copy return tags dict to prevent modifying of inptags: if id(tags) == id(inptags): diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 3425667c..125706af 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase): delattr(self.__action, 'ac') # produce self-referencing query except: self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", - lambda: self.__action.replaceTag(">>>>>>>>>>>>>>>>>>>>", + lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) From e569281d6b9880fdf3d8084dae295880f6700897 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:08:04 +0200 Subject: [PATCH 064/185] avoids overwrite of `known/option` with unmodified (not available) value of `option` from .local config file, so it wouldn't cause self-recursion if `option` already has a reference to `known/option` (from some include) in .conf file; closes gh-2751 --- fail2ban/client/configparserinc.py | 7 ++++--- fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/files/filter.d/testcase02.conf | 12 ++++++++++++ fail2ban/tests/files/filter.d/testcase02.local | 4 ++++ 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 fail2ban/tests/files/filter.d/testcase02.conf create mode 100644 fail2ban/tests/files/filter.d/testcase02.local diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index e0f39579..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -372,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 8abfd4a5..bb42b7a0 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -562,6 +562,17 @@ class FilterReaderTest(LogCaptureTestCase): c = filterReader.convert() self.assertSortedEqual(c, output) + def testFilterReaderSubstKnown(self): + # testcase02.conf + testcase02.local, test covering that known/option is not overridden + # with unmodified (not available) value of option from .local config file, so wouldn't + # cause self-recursion if option already has a reference to known/option in .conf file. + filterReader = FilterReader('testcase02', "jailname", {}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined() + self.assertTrue('sshd' in opts['failregex']) + def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, diff --git a/fail2ban/tests/files/filter.d/testcase02.conf b/fail2ban/tests/files/filter.d/testcase02.conf new file mode 100644 index 00000000..99b3bb45 --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.conf @@ -0,0 +1,12 @@ +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + +[Definition] + +_daemon = sshd +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? + +failregex = %(__prefix_line)s test \ No newline at end of file diff --git a/fail2ban/tests/files/filter.d/testcase02.local b/fail2ban/tests/files/filter.d/testcase02.local new file mode 100644 index 00000000..bfc81d4b --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.local @@ -0,0 +1,4 @@ +[Definition] + +# no options here, coverage for testFilterReaderSubstKnown: +# avoid to overwrite known/option with unmodified (not available) value of option from .local config file \ No newline at end of file From be3115cda0c5ab7def871256995974880221683d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 13:31:29 +0200 Subject: [PATCH 065/185] fix year overflow (9999) by format of datetime (time2str for end of ban of persistent ticket); closes gh-2817 --- fail2ban/server/mytime.py | 7 +++++-- fail2ban/tests/banmanagertestcase.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 98b69bd4..e4b091a7 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -121,8 +121,11 @@ class MyTime: @return ISO-capable string representation of given unixTime """ - return datetime.datetime.fromtimestamp( - unixTime).replace(microsecond=0).strftime(format) + # consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ) + dt = datetime.datetime.fromtimestamp( + unixTime).replace(microsecond=0 + ) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59) + return dt.strftime(format) ## precreate/precompile primitives used in str2seconds: diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a5b37ef6..3be31bc5 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -154,6 +154,21 @@ class AddFailure(unittest.TestCase): finally: self.__banManager.setBanTime(btime) + def testBanList(self): + tickets = [ + BanTicket('192.0.2.1', 1167605999.0), + BanTicket('192.0.2.2', 1167605999.0), + ] + tickets[1].setBanTime(-1) + for t in tickets: + self.__banManager.addBanTicket(t) + self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True), + [ + '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59', + '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59' + ] + ) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): From d0d1f8c362eb927a90abd6c86bab31a8a7717a79 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 16:54:18 +0200 Subject: [PATCH 066/185] improve result for get/set prefregex --- fail2ban/server/transmitter.py | 6 ++++-- fail2ban/tests/servertestcase.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index aff9071c..10cfd163 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -277,7 +277,8 @@ class Transmitter: value = command[2] self.__server.setPrefRegex(name, value) if self.__quiet: return - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) @@ -446,7 +447,8 @@ class Transmitter: elif command[1] == "ignorecache": return self.__server.getIgnoreCache(name) elif command[1] == "prefregex": - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index f1b667b1..3b2552dd 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -513,6 +513,9 @@ class Transmitter(TransmitterBase): jail=self.jailName) self.setGetTest("ignorecache", '', None, jail=self.jailName) + def testJailPrefRegex(self): + self.setGetTest("prefregex", "^Test", jail=self.jailName) + def testJailRegex(self): self.jailAddDelRegexTest("failregex", [ From 07fa9f29129f53924f6850f01adeb8fb7c699060 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 17:04:19 +0200 Subject: [PATCH 067/185] fixes gh-2787: allow to match `did not issue MAIL/EXPN/VRFY/ETRN during connection` non-anchored with extra mode (default names may deviate); additionally provides common addr-tag for IPv4/IPv6 (`(?:IPv6:|)`) and test-coverage for IPv6 --- config/filter.d/sendmail-reject.conf | 13 +++++++------ fail2ban/tests/files/logs/sendmail-reject | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index ca171915..e8b766c5 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,19 +21,20 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=(?:IPv6:|), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^rejecting commands from (\S* )?\[(?:IPv6:|)\] due to pre-greeting traffic after \d+ seconds$ - ^(?:\S+ )?\[(?:IPv6:|)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ - ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:|)\]$ + ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$ mdre-normal = -mdre-extra = ^(?:\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$ +mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection mdre-aggressive = %(mdre-extra)s diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index f69e4531..99c1877c 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -103,3 +103,7 @@ Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52 # failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } +Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 +# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file From 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 17:44:25 +0200 Subject: [PATCH 068/185] filter `sendmail-auth` extended to follow new authentication failure message introduced in sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757) --- config/filter.d/sendmail-auth.conf | 5 +++-- fail2ban/tests/files/logs/sendmail-auth | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 14995fed..c15a2e0c 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -9,10 +9,11 @@ before = common.conf _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) # "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:|)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - +failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index a7ddd6f8..93bf0b14 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -17,3 +17,8 @@ Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SM # failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" } +Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) +# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } +Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) From db1f3477cc415e5b89191b25f36406613012fc6d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 18:07:42 +0200 Subject: [PATCH 069/185] amend to 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a: sendmail-auth has 2 failregex now, so rewritten with prefregex --- config/filter.d/sendmail-auth.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index c15a2e0c..84fcbdda 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -8,12 +8,14 @@ before = common.conf [Definition] _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) +# "\w{14,20}" will give support for IDs from 14 up to 20 characters long __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? addr = (?:IPv6:|) -# "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ +prefregex = ^%(__prefix_line)s.+$ + +failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service From 17a6ba44b3fc9e60b6905ea2694a56f2c58c783b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Aug 2020 13:26:26 +0200 Subject: [PATCH 070/185] fail2ban-regex: speedup formatted output (bypass unneeded stats creation); fail2ban-regex: extended with prefregex statistic --- ChangeLog | 3 +++ fail2ban/client/fail2banregex.py | 43 ++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index cacfdee7..c7f9c759 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,6 +62,9 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) ### New Features and Enhancements +* fail2ban-regex: + - speedup formatted output (bypass unneeded stats creation) + - extended with prefregex statistic * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 98fd9799..1adf9761 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -252,6 +252,8 @@ class Fail2banRegex(object): self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -453,19 +455,33 @@ class Fail2banRegex(object): lines = [] ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False + return None, 0, None if self._filter.getMaxLines() > 1: for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -651,7 +667,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) From a3a148078e1875fd7b33daa1f07e0bb0c2000342 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Aug 2020 14:12:57 +0200 Subject: [PATCH 071/185] fail2ban-regex: more informative output for `datepattern` (e. g. set from filter) - pattern : description, example: Use datepattern : ^%Y-%m-%d %H:%M:%S : ^Year-Month-Day 24hour:Minute:Second --- ChangeLog | 1 + fail2ban/client/fail2banregex.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c7f9c759..fabfed68 100644 --- a/ChangeLog +++ b/ChangeLog @@ -65,6 +65,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * fail2ban-regex: - speedup formatted output (bypass unneeded stats creation) - extended with prefregex statistic + - more informative output for `datepattern` (e. g. set from filter) - pattern : description * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 1adf9761..d0fe55dc 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -294,8 +294,8 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - self.output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: From f09ba1b3993c804aaa279b4542fa8ffd87c4dd53 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Sep 2020 19:59:27 +0200 Subject: [PATCH 072/185] action in jail-config extended to consider space as separator now (splitWithOptions separates by space between mains words, but not in options), so defining `action = a b` would specify 2 actions `a` and `b`; it is additionally more precise now (see fixed typo with closed bracket `]` instead of comma in testServerReloadTest) --- ChangeLog | 2 ++ fail2ban/helpers.py | 2 +- fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/fail2banclienttestcase.py | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index fabfed68..8e9b7612 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition - speedup formatted output (bypass unneeded stats creation) - extended with prefregex statistic - more informative output for `datepattern` (e. g. set from filter) - pattern : description +* parsing of action in jail-configs considers space between action-names as separator also + (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index dc7852ae..f381576e 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -373,7 +373,7 @@ OPTION_EXTRACT_CRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( - r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL) + r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) def extractOptions(option): match = OPTION_CRE.match(option) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index bb42b7a0..2cfaff77 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -264,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) + def testSplitWithOptions(self): + # covering all separators - new-line and spaces: + for sep in ('\n', '\t', ' '): + self.assertEqual(splitWithOptions('a%sb' % (sep,)), ['a', 'b']) + self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)), ['a[x=y]', 'b']) + self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b']) + self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)), ['a[x="y z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b']) + def testIncorrectJail(self): jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index bbd6964a..03b1d7ce 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -856,7 +856,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: '," " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, From ed20d457b21d298cf8a8569ab2eb09546e8065c4 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Sep 2020 20:14:31 +0200 Subject: [PATCH 073/185] jail.conf: removed action parameter `name` that set on jail-name (`name=%(__name__)s` is default in action reader) --- config/jail.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 40827707..8fbd23a1 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -171,16 +171,16 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. action_mw = %(action_)s - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(action_)s - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # @@ -192,7 +192,7 @@ action_xarf = %(action_)s # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # From 70c601e9e53a44a6a2e950ee8e64705d14f09bc7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 2 Sep 2020 20:47:05 +0200 Subject: [PATCH 074/185] involve config parameter (replaces hard-coded path); fixed typo in actionban (looks like copy&paste from trimmed tty) --- config/action.d/apprise.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf index ac54d8fd..37c42ea2 100644 --- a/config/action.d/apprise.conf +++ b/config/action.d/apprise.conf @@ -10,13 +10,13 @@ # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = printf %%b "The jail as been started successfully." |apprise -t "[Fail2Ban] : started on `uname -n`" -c /etc/fail2ban/apprise.conf +actionstart = printf %%b "The jail as been started successfully." | -t "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = printf %%b "The jail has been stopped." |apprise -t "[Fail2Ban] : stopped on `uname -n`" -c /etc/fail2ban/apprise.conf +actionstop = printf %%b "The jail has been stopped." | -t "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -30,7 +30,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | apprise -n "warning" -t "[Fail2Ban] : banned $ +actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | -n "warning" -t "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -45,3 +45,5 @@ actionunban = # Define location of the default apprise configuration file to use # config = /etc/fail2ban/apprise.conf +# +apprise = apprise -c "" From a038fd5dfe8cb0714472833604735b83462a217d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 3 Sep 2020 16:41:23 +0200 Subject: [PATCH 075/185] `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-`; small optimizations on `firewallcmd-rich-rules.conf` and `firewallcmd-rich-logging.conf` simplifying both and provide a dependency (rich-logging is a derivative of rich-rules); closes gh-2821 --- ChangeLog | 1 + config/action.d/firewallcmd-ipset.conf | 2 +- config/action.d/firewallcmd-multiport.conf | 4 +- config/action.d/firewallcmd-new.conf | 4 +- config/action.d/firewallcmd-rich-logging.conf | 30 ++---------- config/action.d/firewallcmd-rich-rules.conf | 8 ++-- fail2ban/tests/servertestcase.py | 48 +++++++++++++++---- 7 files changed, 55 insertions(+), 42 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8e9b7612..361b81d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -51,6 +51,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing with `jq`, gh-2140, gh-2656) * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 42513933..66358f23 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -69,7 +69,7 @@ allports = -p # Option: multiport # Notes.: addition to block access only to specific ports # Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -multiport = -p -m multiport --dports +multiport = -p -m multiport --dports "$(echo '' | sed s/:/-/g)" ipmset = f2b- familyopt = diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf index 81540e5b..0c401f1b 100644 --- a/config/action.d/firewallcmd-multiport.conf +++ b/config/action.d/firewallcmd-multiport.conf @@ -11,9 +11,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index b06f5ccd..7b08603c 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -10,9 +10,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index badfee83..21e45087 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -1,6 +1,6 @@ # Fail2Ban configuration file # -# Author: Donald Yandt +# Authors: Donald Yandt, Sergey G. Brester # # Because of the rich rule commands requires firewalld-0.3.1+ # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not @@ -10,36 +10,15 @@ # # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/.xml that can be shared and parsed easliy # -# Example commands to view rules: -# firewall-cmd [--zone=] --list-rich-rules -# firewall-cmd [--zone=] --list-all -# firewall-cmd [--zone=zone] --query-rich-rule='rule' +# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters. [INCLUDES] -before = firewallcmd-common.conf +before = firewallcmd-rich-rules.conf [Definition] -actionstart = - -actionstop = - -actioncheck = - -# you can also use zones and/or service names. -# -# zone example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' port port='' protocol='' log prefix='f2b-' level='' limit value='/m' " -# -# service name example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' service name='' log prefix='f2b-' level='' limit value='/m' " -# -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp - -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done - -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done +rich-suffix = log prefix='f2b-' level='' limit value='/m' [Init] @@ -48,4 +27,3 @@ level = info # log rate per minute rate = 1 - diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index bed71797..803d7d12 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -35,8 +35,10 @@ actioncheck = # # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done +fwcmd_rich_rule = rule family='' source address='' port port='$p' protocol='' %(rich-suffix)s + +actionban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done - +actionunban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done +rich-suffix = \ No newline at end of file diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 3b2552dd..eaf1b346 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1780,18 +1780,18 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`", ), @@ -1859,21 +1859,21 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", ), 'ip6-start': ( "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", ), 'flush': ( "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset flush f2b-j-w-fwcmd-ipset6`", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset destroy f2b-j-w-fwcmd-ipset`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset6`", "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), @@ -1926,6 +1926,38 @@ class ServerConfigReaderTests(LogCaptureTestCase): r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", ), }), + # firewallcmd-rich-rules -- + ('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + }), + # firewallcmd-rich-logging -- + ('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + }), ) server = TestServer() transm = server._Server__transm From f555ff45e99c804a3fa48a2db1af56fa9200da0a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 19:08:52 +0200 Subject: [PATCH 076/185] attempt to speedup ban- and fail-manager (e. g. fail2ban-client status, see gh-2819), remove unneeded lock (GIL is enough here) --- fail2ban/server/banmanager.py | 20 +++++++------------- fail2ban/server/failmanager.py | 9 +++------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 479ba26f..fe38dec6 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -66,8 +66,7 @@ class BanManager: # @param value the time def setBanTime(self, value): - with self.__lock: - self.__banTime = int(value) + self.__banTime = int(value) ## # Get the ban time. @@ -76,8 +75,7 @@ class BanManager: # @return the time def getBanTime(self): - with self.__lock: - return self.__banTime + return self.__banTime ## # Set the total number of banned address. @@ -85,8 +83,7 @@ class BanManager: # @param value total number def setBanTotal(self, value): - with self.__lock: - self.__banTotal = value + self.__banTotal = value ## # Get the total number of banned address. @@ -94,8 +91,7 @@ class BanManager: # @return the total number def getBanTotal(self): - with self.__lock: - return self.__banTotal + return self.__banTotal ## # Returns a copy of the IP list. @@ -103,8 +99,7 @@ class BanManager: # @return IP list def getBanList(self): - with self.__lock: - return list(self.__banList.keys()) + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,9 +107,8 @@ class BanManager: # @return ban list iterator def __iter__(self): - # ensure iterator is safe (traverse over the list in snapshot created within lock): - with self.__lock: - return iter(list(self.__banList.values())) + # ensure iterator is safe - traverse over the list in snapshot created within lock (GIL): + return iter(list(self.__banList.values())) ## # Returns normalized value diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3458aed5..3e81e8b5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -47,12 +47,10 @@ class FailManager: self.__bgSvc = BgService() def setFailTotal(self, value): - with self.__lock: - self.__failTotal = value + self.__failTotal = value def getFailTotal(self): - with self.__lock: - return self.__failTotal + return self.__failTotal def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... @@ -123,8 +121,7 @@ class FailManager: return attempts def size(self): - with self.__lock: - return len(self.__failList) + return len(self.__failList) def cleanup(self, time): with self.__lock: From 5abc4ba4ae280b6ec89c6bffc9dd2140d0d56dc4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 22:11:51 +0200 Subject: [PATCH 077/185] amend to 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c (#2758): better reaction on broken pipe (on long output), don't close stdout explicitly (allows usage of modules like cProfile, which outputs result on exit), just flush it before exit. --- fail2ban/client/fail2bancmdline.py | 24 ++++++++++++++++-------- fail2ban/helpers.py | 5 +++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 8936e03f..03683cad 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,13 +27,17 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise # Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") @@ -310,12 +314,16 @@ class Fail2banCmdLine(): def _exit(code=0): # implicit flush without to produce broken pipe error (32): sys.stderr.close() - sys.stdout.close() - # exit: - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f381576e..c45be849 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -224,9 +224,10 @@ def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover sys.exit(0) try: - BrokenPipeError + BrokenPipeError = BrokenPipeError except NameError: # pragma: 3.x no cover - BrokenPipeError = IOError + BrokenPipeError = IOError + __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, From e8ee3ba544c7eefd7b41e34ec3a44550c1280fbb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:36:54 +0200 Subject: [PATCH 078/185] resolves a bottleneck within transmitting of large data between server and client: speedup search of communications end-marker and increase max buffer size (up to 32KB) --- fail2ban/client/csocket.py | 14 +++++++++----- fail2ban/tests/sockettestcase.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 8cd22a41..2d414e5c 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -152,7 +152,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(RuntimeError, r"socket connection broken", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler @@ -168,7 +168,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: TestMsgError() - #self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage) + #self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage) self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error') finally: RequestHandler.found_terminator = org_handler From f381b982465e3e9bf58f565cfea006fedc1989f1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:44:55 +0200 Subject: [PATCH 079/185] introduces new flavor `short` for `fail2ban-client status $jail short`: output total and current counts only, without banned IPs list in order to speedup it and to provide more clear output (gh-2819), flavor `basic` (still default) is unmodified for backwards compatibility; it can be changed later to `short`, so for full list of IPs in newer version one should better use: - `fail2ban-client status $jail basic` - `fail2ban-client get $jail banned` or `fail2ban-client banned` --- fail2ban/server/actions.py | 14 ++++++++++---- fail2ban/tests/actionstestcase.py | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3308d4b2..f14d8d7b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -661,13 +661,19 @@ class Actions(JailThread, Mapping): """Status of current and total ban counts and current banned IP list. """ # TODO: Allow this list to be printed as 'status' output - supported_flavors = ["basic", "cymru"] + supported_flavors = ["short", "basic", "cymru"] if flavor is None or flavor not in supported_flavors: logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) - ret = [("Currently banned", self.__banManager.size()), - ("Total banned", self.__banManager.getBanTotal()), - ("Banned IP list", self.__banManager.getBanList())] + if flavor != "short": + banned = self.__banManager.getBanList() + cnt = len(banned) + else: + cnt = self.__banManager.size() + ret = [("Currently banned", cnt), + ("Total banned", self.__banManager.getBanTotal())] + if flavor != "short": + ret += [("Banned IP list", banned)] if flavor == "cymru": cymru_info = self.__banManager.getBanListExtendedCymruInfo() ret += \ diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index d97d9921..532fe6ed 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase): self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop') self.assertEqual(self.__actions.status(),[("Currently banned", 0 ), ("Total banned", 0 ), ("Banned IP list", [] )]) + self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ), + ("Total banned", 0 )]) def testAddActionPython(self): self.__actions.add( From d977d81ef75f96d41e8aa994aa8efd81f13bf795 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 17 Sep 2020 12:39:08 +0200 Subject: [PATCH 080/185] action.d/abuseipdb.conf: removed broken link, simplified usage example, fixed typos --- config/action.d/abuseipdb.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index d36cb3a9..ed958c86 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -21,14 +21,13 @@ # # Example, for ssh bruteforce (in section [sshd] of `jail.local`): # action = %(known/action)s -# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] # -# See below for catagories. +# See below for categories. # -# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) -## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call. # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] # ID Title Description # 3 Fraud Orders @@ -101,5 +100,5 @@ actionunban = # Notes Your API key from abuseipdb.com # Values: STRING Default: None # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. -# You will need to set the catagory in the action call. +# You will need to set the category in the action call. abuseipdb_apikey = From f518d42c590f4ea3d8632937dc6ad7e875419b00 Mon Sep 17 00:00:00 2001 From: Nathan Henrie Date: Wed, 12 Aug 2020 14:11:33 -0600 Subject: [PATCH 081/185] Add a note about `journalflags` options to `systemd-journal` backend Also adds systemd backend configuration examples to jail.conf(5) Closes #2696 --- fail2ban/client/fail2banregex.py | 4 +++- man/jail.conf.5 | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index d0fe55dc..56da17dd 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -113,7 +113,9 @@ class _f2bOptParser(OptionParser): LOG: string a string representing a log line filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + "systemd-journal" search systemd journal. Optionally specify + `systemd-journal[journalflags=X]` to determine + which journals are used (systemd-python required) REGEX: string a string representing a 'failregex' diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 830c8aed..d7722124 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst uses a polling algorithm which does not require external libraries. .TP .B systemd -uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples: +.PP +.RS +.nf +backend = systemd[journalpath=/run/log/journal/machine-1] +backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"] +backend = systemd[journalflags=1] +.fi .SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: From 24093de32daa05af5f178639095ac7106d55b544 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 23 Sep 2020 19:35:17 +0200 Subject: [PATCH 082/185] small amend (simplifying formatted help and man) --- fail2ban/client/fail2banregex.py | 22 +++++++++++----------- man/fail2ban-regex.1 | 9 +++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 56da17dd..e7a4e214 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,7 +21,6 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" @@ -109,21 +108,22 @@ class _f2bOptParser(OptionParser): def format_help(self, *args, **kwargs): """ Overwritten format helper with full ussage.""" self.usage = '' - return "Usage: " + usage() + __doc__ + """ + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal. Optionally specify - `systemd-journal[journalflags=X]` to determine - which journals are used (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n Report bugs to https://github.com/fail2ban/fail2ban/issues\n """ + __copyright__ + "\n" diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index bb89ef8c..3964126f 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -18,13 +18,18 @@ a string representing a log line filename path to a log file (\fI\,/var/log/auth.log\/\fP) .TP -"systemd\-journal" -search systemd journal (systemd\-python required) +systemd\-journal +search systemd journal (systemd\-python required), +optionally with backend parameters, see `man jail.conf` +for usage and examples (systemd\-journal[journalflags=1]). .SS "REGEX:" .TP string a string representing a 'failregex' .TP +filter +name of filter, optionally with options (sshd[mode=aggressive]) +.TP filename path to a filter file (filter.d/sshd.conf) .SS "IGNOREREGEX:" From 1418bcdf5bb01dd210f4d37b5c2de8ccd0c658e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 12:35:49 +0200 Subject: [PATCH 083/185] `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num`, exit code can't be larger than 255 (gh-2836) --- ChangeLog | 1 + config/action.d/bsd-ipfw.conf | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 361b81d5..c3e2c6d4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition with `jq`, gh-2140, gh-2656) * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) * `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) +* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 5116b0d8..7f04fe7c 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,10 @@ # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipfw show | fgrep -c -m 1 -s 'table()' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num from table\(
\) to me ; echo $num > "" ) +actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" + ) # Option: actionstop From 2817a8144c685d3ca18b7ca27064179ac318924e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 13:33:40 +0200 Subject: [PATCH 084/185] `action.d/bsd-ipfw.conf`: small amend (gh-2836) simplifying awk condition/code (position starts from `` and increases whilst used) --- config/action.d/bsd-ipfw.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 7f04fe7c..444192d3 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -15,7 +15,7 @@ # Values: CMD # actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( - num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 == b) { b = $1 + 1 } } END { print b }'); ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" ) From c8059bf9b35a167ad7bd4579f83495f9dda50c79 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 16:27:17 +0200 Subject: [PATCH 085/185] ban/unban: increase responsiveness of actions thread by (un)banning process, better waiting timeout considering pending tickets for unban (_nextUnbanTime) --- fail2ban/server/actions.py | 46 ++++++++++++++++++++--------------- fail2ban/server/banmanager.py | 16 +++++------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index f14d8d7b..b7b95b44 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -327,25 +327,33 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if self.idle: - logSys.debug("Actions: enter idle mode") - Utils.wait_for(lambda: not self.active or not self.idle, - lambda: False, self.sleeptime) - logSys.debug("Actions: leave idle mode") - continue - # wait for ban (stop if gets inactive): - bancnt = 0 - if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, self.sleeptime): - bancnt = self.__checkBan() - cnt += bancnt - # unban if nothing is banned not later than banned tickets >= banPrecedence - if not bancnt or cnt >= self.banPrecedence: - if self.active: - # let shrink the ban list faster - bancnt *= 2 - self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) - cnt = 0 - + try: + if self.idle: + logSys.debug("Actions: enter idle mode") + Utils.wait_for(lambda: not self.active or not self.idle, + lambda: False, self.sleeptime) + logSys.debug("Actions: leave idle mode") + continue + # wait for ban (stop if gets inactive, pending ban or unban): + bancnt = 0 + wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): + bancnt = self.__checkBan() + cnt += bancnt + # unban if nothing is banned not later than banned tickets >= banPrecedence + if not bancnt or cnt >= self.banPrecedence: + if self.active: + # let shrink the ban list faster + bancnt *= 2 + logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount) + self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) + cnt = 0 + except Exception as e: # pragma: no cover + logSys.error("[%s] unhandled error in actions thread: %s", + self._jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + self.__flushBan(stop=True) self.stopActions() return True diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index fe38dec6..575d648b 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -57,7 +57,7 @@ class BanManager: ## Total number of banned IP address self.__banTotal = 0 ## The time for next unban process (for performance and load reasons): - self.__nextUnbanTime = BanTicket.MAX_TIME + self._nextUnbanTime = BanTicket.MAX_TIME ## # Set the ban time. @@ -290,8 +290,8 @@ class BanManager: self.__banList[fid] = ticket self.__banTotal += 1 # correct next unban time: - if self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if self._nextUnbanTime > eob: + self._nextUnbanTime = eob return True ## @@ -322,12 +322,8 @@ class BanManager: def unBanList(self, time, maxCount=0x7fffffff): with self.__lock: - # Permanent banning - if self.__banTime < 0: - return list() - # Check next unban time: - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime if nextUnbanTime > time: return list() @@ -340,12 +336,12 @@ class BanManager: if time > eob: unBanList[fid] = ticket if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime break elif nextUnbanTime > eob: nextUnbanTime = eob - self.__nextUnbanTime = nextUnbanTime + self._nextUnbanTime = nextUnbanTime # Removes tickets. if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: From 02525d7b6f96c2841c24c4dbfcfe14e4a3d1ef63 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 8 Oct 2020 21:07:51 +0200 Subject: [PATCH 086/185] filter.d/sshd.conf: mode `ddos` (and `aggressive`) extended with new rule closing flood attack vector, matching: error: kex_exchange_identification: Connection closed by remote host (gh-2850) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 4c86dca0..e7942262 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -71,7 +71,7 @@ mdre-normal = mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from - ^kex_exchange_identification: client sent invalid protocol identifier + ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host) ^Bad protocol version identification '.*' from ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 9fff416a..5d23f96f 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -312,6 +312,8 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 # failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" +# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" } +Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] From a07e6fe1a2d1b8945d8a84abf3022061c9cc4528 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 30 Oct 2020 13:19:56 +0100 Subject: [PATCH 087/185] reduce default `maxmatches` from 50 to 5: avoid too large memory consumption if `maxretry` is large and many failures don't cause ban (but accumulated in fail-manager with all the matched lines); closes gh-2843 --- fail2ban/server/failmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3e81e8b5..4173a233 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -43,7 +43,7 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 - self.maxMatches = 50 + self.maxMatches = 5 self.__bgSvc = BgService() def setFailTotal(self, value): From 9d77fb2b4c5f4471e1abef0974c70c1ff622ff75 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 3 Nov 2020 13:15:42 +0100 Subject: [PATCH 088/185] 1st try of GH actions flow (CI only, no coverage atm) --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..154a75ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + python -m pip install --upgrade pip + pip install 2to3 + fi + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 7cb6412f68afd2590bb86d3d9939e0d21ad00367 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 3 Nov 2020 13:15:42 +0100 Subject: [PATCH 089/185] 1st try of GH actions flow (CI only, no coverage atm) --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..154a75ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + python -m pip install --upgrade pip + pip install 2to3 + fi + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 7f0010be68d0ca778ae97698bdd893c859b282eb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 3 Nov 2020 15:51:49 +0100 Subject: [PATCH 090/185] attempt to install systemd-python module --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 154a75ae..7a1d31df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,10 +44,12 @@ jobs: - name: Install dependencies run: | + python -m pip install --upgrade pip if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then - python -m pip install --upgrade pip pip install 2to3 fi + pip install systemd-python || echo 'systemd not available' + pip install pyinotify || echo 'inotify not available' - name: Before scripts run: | From 55d6408b13c75100134f123cc283a801a384fd6c Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Nov 2020 15:31:11 +0100 Subject: [PATCH 091/185] tweaks to speedup test-cases (test-suite seems to be time stable now, so we could shorten sleeping intervals) --- fail2ban/tests/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 47e5b909..b54581f5 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -292,15 +292,15 @@ def initTests(opts): unittest.F2B.SkipIfFast = F2B_SkipIfFast else: # smaller inertance inside test-cases (litle speedup): - Utils.DEFAULT_SLEEP_TIME = 0.25 - Utils.DEFAULT_SLEEP_INTERVAL = 0.025 + Utils.DEFAULT_SLEEP_TIME = 0.025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.005 Utils.DEFAULT_SHORT_INTERVAL = 0.0005 # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover + if v > 0.25: # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) - _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + _org_sleep(v) time.sleep = _new_sleep # --no-network : if unittest.F2B.no_network: # pragma: no cover From e700ccc6671d64aa725905763c66d74522bbd6ca Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Nov 2020 16:51:49 +0100 Subject: [PATCH 092/185] filter apache-modsecurity: added coverage for different log-format (apache 2.4 and php-fpm, gh-2717) --- fail2ban/tests/files/logs/apache-modsecurity | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index dbb14863..7e2f8c86 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -6,3 +6,6 @@ # failJSON: { "time": "2018-09-28T09:18:06", "match": true , "host": "192.0.2.1", "desc": "two client entries in message (gh-2247)" } [Sat Sep 28 09:18:06 2018] [error] [client 192.0.2.1:55555] [client 192.0.2.1] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] + +# failJSON: { "time": "2020-05-09T00:35:52", "match": true , "host": "192.0.2.2", "desc": "new format - apache 2.4 and php-fpm (gh-2717)" } +[Sat May 09 00:35:52.389262 2020] [:error] [pid 22406:tid 139985298601728] [client 192.0.2.2:47762] [client 192.0.2.2] ModSecurity: Access denied with code 401 (phase 2). Operator EQ matched 1 at IP:blocked. [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_wp_login.conf"] [line "14"] [id "500000"] [msg "Ip address blocked for 15 minutes, more than 5 login attempts in 3 minutes."] [hostname "example.com"] [uri "/wp-login.php"] [unique_id "XrYlGL5IY3I@EoLOgAAAA8"], referer: https://example.com/wp-login.php From ec873e2dc38af51d7da41d65fb28a1daf707b557 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Thu, 5 Nov 2020 23:56:30 +0100 Subject: [PATCH 093/185] Add SoftEtherVPN jail --- ChangeLog | 1 + config/filter.d/softethervpn.conf | 9 +++++++++ config/jail.conf | 5 +++++ fail2ban/tests/files/logs/softethervpn | 7 +++++++ 4 files changed, 22 insertions(+) create mode 100644 config/filter.d/softethervpn.conf create mode 100644 fail2ban/tests/files/logs/softethervpn diff --git a/ChangeLog b/ChangeLog index c3e2c6d4..d1aa66c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * parsing of action in jail-configs considers space between action-names as separator also (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf new file mode 100644 index 00000000..0cbf5c94 --- /dev/null +++ b/config/filter.d/softethervpn.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for SoftEtherVPN +# Detecting unauthorized access to SoftEtherVPN +# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration + +[INCLUDES] +before = common.conf + +[Definition] +failregex = ^%(__prefix_line)s(?:\([0-9 :.-]{23}\) :)? Connection ".+": User authentication failed. The user name that has been provided was ".+", from .$ diff --git a/config/jail.conf b/config/jail.conf index 8fbd23a1..67f39e40 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -820,6 +820,11 @@ udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010 action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + [gitlab] port = http,https logpath = /var/log/gitlab/gitlab-rails/application.log diff --git a/fail2ban/tests/files/logs/softethervpn b/fail2ban/tests/files/logs/softethervpn new file mode 100644 index 00000000..dd2a798b --- /dev/null +++ b/fail2ban/tests/files/logs/softethervpn @@ -0,0 +1,7 @@ +# Access of unauthorized host in /usr/local/vpnserver/security_log/*/sec.log +# failJSON: { "time": "2020-05-12T10:53:19", "match": true , "host": "80.10.11.12" } +2020-05-12 10:53:19.781 Connection "CID-72": User authentication failed. The user name that has been provided was "bob", from 80.10.11.12. + +# Access of unauthorized host in syslog +# failJSON: { "time": "2020-05-13T10:53:19", "match": true , "host": "80.10.11.13" } +2020-05-13T10:53:19 localhost [myserver.com/VPN/defaultvpn] (2020-05-13 10:53:19.591) : Connection "CID-594": User authentication failed. The user name that has been provided was "alice", from 80.10.11.13. From 5430091acb5ba0ec7cc9cbb50a4346ec9e5c896a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2020 12:43:34 +0100 Subject: [PATCH 094/185] jail `counter-strike`: removed link to site with redirect to malicious page (gh-2868) --- config/jail.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index fb76926d..b0a67720 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -789,7 +789,6 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] From 010e76406fd2aac83b8cf6da27e9d380cc75dad4 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 9 Nov 2020 13:19:25 +0100 Subject: [PATCH 095/185] small tweaks (both 2nd time and facility are optional, avoid catch-all, etc) --- config/filter.d/softethervpn.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf index 0cbf5c94..f7e7c0c3 100644 --- a/config/filter.d/softethervpn.conf +++ b/config/filter.d/softethervpn.conf @@ -6,4 +6,4 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(?:\([0-9 :.-]{23}\) :)? Connection ".+": User authentication failed. The user name that has been provided was ".+", from .$ +failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "(?:[^"]+|.+)", from \.$ From df659a0cbc68ad7f8233f16edf64ddddec6dd1d7 Mon Sep 17 00:00:00 2001 From: Mart124 <37041094+Mart124@users.noreply.github.com> Date: Sun, 18 Oct 2020 19:56:30 +0200 Subject: [PATCH 096/185] Add Bitwarden syslog support --- ChangeLog | 1 + config/filter.d/bitwarden.conf | 8 +++++++- fail2ban/tests/files/logs/bitwarden | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d1aa66c5..96c58bb5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -73,6 +73,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * new filter and jail for GitLab recognizing failed application logins (gh-2689) * new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) +* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf index 29bd4be8..4a836cbb 100644 --- a/config/filter.d/bitwarden.conf +++ b/config/filter.d/bitwarden.conf @@ -2,5 +2,11 @@ # Detecting failed login attempts # Logged in bwdata/logs/identity/Identity/log.txt +[INCLUDES] +before = common.conf + [Definition] -failregex = ^\s*\[WRN\]\s+Failed login attempt(?:, 2FA invalid)?\. $ +failregex = ^%(__prefix_line)s\s*\[[^\s]+\]\s+Failed login attempt(?:, 2FA invalid)?\. $ + +# DEV Notes: +# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 3642b3bf..9deb2c07 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -3,3 +3,6 @@ # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } 2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } +2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 From 2a18b82f5f92ca50b63dcb01b6f4231cd4220f9f Mon Sep 17 00:00:00 2001 From: Mart124 <37041094+Mart124@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:18:03 +0200 Subject: [PATCH 097/185] Support alternative Bitwarden log format --- fail2ban/tests/files/logs/bitwarden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 9deb2c07..27a22854 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -2,7 +2,7 @@ 2019-11-26 01:04:49.008 +08:00 [WRN] Failed login attempt. 192.168.0.16 # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } -2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 +2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 # failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } 2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 From 25e006e137172c96c25864f8050b191efaaba3d8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2020 13:43:59 +0100 Subject: [PATCH 098/185] review and small tweaks (more precise and safe RE) --- config/filter.d/bitwarden.conf | 3 ++- fail2ban/tests/files/logs/bitwarden | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf index 4a836cbb..b0651c8e 100644 --- a/config/filter.d/bitwarden.conf +++ b/config/filter.d/bitwarden.conf @@ -6,7 +6,8 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s\s*\[[^\s]+\]\s+Failed login attempt(?:, 2FA invalid)?\. $ +_daemon = Bitwarden-Identity +failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. $ # DEV Notes: # __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 27a22854..0fede6c6 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -1,6 +1,9 @@ # failJSON: { "time": "2019-11-25T18:04:49", "match": true , "host": "192.168.0.16" } 2019-11-26 01:04:49.008 +08:00 [WRN] Failed login attempt. 192.168.0.16 +# failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } +2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } 2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 From 840f0ff10a5edb14afaba8b2c13bc18d2514715d Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:31:06 +0100 Subject: [PATCH 099/185] Add Grafana jail --- ChangeLog | 1 + config/filter.d/grafana.conf | 9 +++++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/grafana | 5 +++++ 4 files changed, 19 insertions(+) create mode 100644 config/filter.d/grafana.conf create mode 100644 fail2ban/tests/files/logs/grafana diff --git a/ChangeLog b/ChangeLog index 96c58bb5..51ba4f90 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * parsing of action in jail-configs considers space between action-names as separator also (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for Grafana recognizing failed application logins (gh-2855) * new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf new file mode 100644 index 00000000..78ded075 --- /dev/null +++ b/config/filter.d/grafana.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for Grafana +# Detecting unauthorized access +# Typically logged in /var/log/grafana/grafana.log + +[Init] +datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z + +[Definition] +failregex = ^.*msg="Invalid username or password".* remote_addr=$ diff --git a/config/jail.conf b/config/jail.conf index 5ca67749..ddbcf61e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -828,6 +828,10 @@ logpath = /usr/local/vpnserver/security_log/*/sec.log port = http,https logpath = /var/log/gitlab/gitlab-rails/application.log +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + [bitwarden] port = http,https logpath = /home/*/bwdata/logs/identity/Identity/log.txt diff --git a/fail2ban/tests/files/logs/grafana b/fail2ban/tests/files/logs/grafana new file mode 100644 index 00000000..aac86ebc --- /dev/null +++ b/fail2ban/tests/files/logs/grafana @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/grafana/grafana.log +# failJSON: { "time": "2020-10-19T17:44:33", "match": true , "host": "182.56.23.12" } +t=2020-10-19T17:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="Invalid Username or Password" remote_addr=182.56.23.12 +# failJSON: { "time": "2020-10-19T18:44:33", "match": true , "host": "182.56.23.13" } +t=2020-10-19T18:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="User not found" remote_addr=182.56.23.13 From 1c1a9b868c6c7677c929ede5e8c1cb8f16d3be41 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 9 Nov 2020 15:36:30 +0100 Subject: [PATCH 100/185] no catch-alls, user name and error message stored in ticket --- config/filter.d/grafana.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf index 78ded075..e7f0f420 100644 --- a/config/filter.d/grafana.conf +++ b/config/filter.d/grafana.conf @@ -6,4 +6,4 @@ datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z [Definition] -failregex = ^.*msg="Invalid username or password".* remote_addr=$ +failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"[^"]+"|\S+)| error="[^"]+"| \S+=(?:\S*|"[^"]+"))* remote_addr=$ From ca4da9d1d3f792562500182711abc149b2c741a1 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 11 Nov 2020 11:08:23 +0100 Subject: [PATCH 101/185] actions: extend tags replacement in non ticket-based commands (actionstart, actionstop, etc); fixes regression by interpolation of tag `` introduced in 0.11 with dynamic bantime (due to `bantime.increment`, see #2869) --- fail2ban/server/action.py | 13 ++++++++++++- fail2ban/server/actions.py | 12 +++++++----- fail2ban/tests/servertestcase.py | 3 ++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 1c313cf0..f52e0878 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -455,7 +455,18 @@ class CommandAction(ActionBase): ret = True # avoid double execution of same command for both families: if cmd and cmd not in self._operationExecuted(tag, lambda f: f != famoper): - ret = self.executeCmd(cmd, self.timeout) + realCmd = cmd + if self._jail: + # simulate action info with "empty" ticket: + aInfo = getattr(self._jail.actions, 'actionInfo', None) + if not aInfo: + aInfo = self._jail.actions._getActionInfo(None) + setattr(self._jail.actions, 'actionInfo', aInfo) + aInfo['time'] = MyTime.time() + aInfo['family'] = famoper + # replace dynamical tags, important - don't cache, no recursion and auto-escape here + realCmd = self.replaceDynamicTags(cmd, aInfo) + ret = self.executeCmd(realCmd, self.timeout) res &= ret if afterExec: afterExec(famoper, ret) self._operationExecuted(tag, famoper, cmd if ret else None) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 49db5df7..967908af 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -457,7 +457,9 @@ class Actions(JailThread, Mapping): return mi[idx] if mi[idx] is not None else self.__ticket - def __getActionInfo(self, ticket): + def _getActionInfo(self, ticket): + if not ticket: + ticket = BanTicket("", MyTime.time()) aInfo = Actions.ActionInfo(ticket, self._jail) return aInfo @@ -491,7 +493,7 @@ class Actions(JailThread, Mapping): bTicket = BanTicket.wrap(ticket) btime = ticket.getBanTime(self.__banManager.getBanTime()) ip = bTicket.getIP() - aInfo = self.__getActionInfo(bTicket) + aInfo = self._getActionInfo(bTicket) reason = {} if self.__banManager.addBanTicket(bTicket, reason=reason): cnt += 1 @@ -568,7 +570,7 @@ class Actions(JailThread, Mapping): """ actions = actions or self._actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Reban %s%s", self._jail.name, aInfo["ip"], (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) for name, action in actions.iteritems(): @@ -602,7 +604,7 @@ class Actions(JailThread, Mapping): if not action._prolongable: continue if aInfo is None: - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if not aInfo.immutable: aInfo.reset() action.prolong(aInfo) except Exception as e: @@ -696,7 +698,7 @@ class Actions(JailThread, Mapping): else: unbactions = actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) for name, action in unbactions.iteritems(): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 3bd4501b..d2bf8bdc 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1448,9 +1448,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # dummy -- - ('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', { + ('j-dummy', '''dummy[name=%(__name__)s, init="=='/'==bt:==bc:==", target="/tmp/fail2ban.dummy"]''', { 'ip4': ('family: inet4',), 'ip6': ('family: inet6',), 'start': ( + '''`printf %b "=='/'==bt:600==bc:0==\\n"''', ## empty family (independent in this action, same for both), no ip on start, initial bantime and bancount '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`', ), 'flush': ( From 1cefb7fdc6f160c300bfb7fe329b4402703cb2a7 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 14:40:48 +0100 Subject: [PATCH 102/185] setup.py: small amend to d2d3762ba9fa82b7983bae74cd567702e5c1b96c: allow build without tests also (both build and install accepting `--without-tests` option now) --- setup.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index ce1eedf6..2e2a77fa 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ source_dir = os.path.realpath(os.path.dirname( sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ )) +with_tests = True + # Wrapper to install python binding (to current python version): class install_scripts_f2b(install_scripts): @@ -123,7 +125,7 @@ class install_command_f2b(install): ] def initialize_options(self): self.disable_2to3 = None - self.without_tests = None + self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): global _2to3 @@ -168,6 +170,12 @@ elif "test" in sys.argv: print("python distribute required to execute fail2ban tests") print("") +# if build without tests: +if "build" in sys.argv: + if "--without-tests" in sys.argv: + with_tests = False + sys.argv.remove("--without-tests") + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -224,16 +232,18 @@ setup( 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper - ], + ] + [ + 'bin/fail2ban-testcases', + ] if with_tests else [], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + ] + [ 'fail2ban.tests', 'fail2ban.tests.action_d', - ], + ] if with_tests else [], package_data = { 'fail2ban.tests': [ join(w[0], f).replace("fail2ban/tests/", "", 1) @@ -245,7 +255,7 @@ setup( [ join(w[0], f).replace("fail2ban/tests/", "", 1) for w in os.walk('fail2ban/tests/action_d') for f in w[2]] - }, + } if with_tests else {}, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From a107a8e7d2f1e55e317425cc12acbd813d9e6dee Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 14:59:45 +0100 Subject: [PATCH 103/185] setup.py: cherry-pick from 0.11 (option --without-tests) --- setup.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 8da29268..2e2a77fa 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ source_dir = os.path.realpath(os.path.dirname( sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ )) +with_tests = True + # Wrapper to install python binding (to current python version): class install_scripts_f2b(install_scripts): @@ -119,9 +121,11 @@ class install_scripts_f2b(install_scripts): class install_command_f2b(install): user_options = install.user_options + [ ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), + ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): self.disable_2to3 = None + self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): global _2to3 @@ -132,6 +136,13 @@ class install_command_f2b(install): cmdclass = self.distribution.cmdclass cmdclass['build_py'] = build_py_2to3 cmdclass['build_scripts'] = build_scripts_2to3 + if self.without_tests: + self.distribution.scripts.remove('bin/fail2ban-testcases') + + self.distribution.packages.remove('fail2ban.tests') + self.distribution.packages.remove('fail2ban.tests.action_d') + + del self.distribution.package_data['fail2ban.tests'] install.finalize_options(self) def run(self): install.run(self) @@ -159,6 +170,12 @@ elif "test" in sys.argv: print("python distribute required to execute fail2ban tests") print("") +# if build without tests: +if "build" in sys.argv: + if "--without-tests" in sys.argv: + with_tests = False + sys.argv.remove("--without-tests") + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -208,23 +225,25 @@ setup( license = "GPL", platforms = "Posix", cmdclass = { - 'build_py': build_py, 'build_scripts': build_scripts, + 'build_py': build_py, 'build_scripts': build_scripts, 'install_scripts': install_scripts_f2b, 'install': install_command_f2b }, scripts = [ 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper - ], + ] + [ + 'bin/fail2ban-testcases', + ] if with_tests else [], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + ] + [ 'fail2ban.tests', 'fail2ban.tests.action_d', - ], + ] if with_tests else [], package_data = { 'fail2ban.tests': [ join(w[0], f).replace("fail2ban/tests/", "", 1) @@ -236,7 +255,7 @@ setup( [ join(w[0], f).replace("fail2ban/tests/", "", 1) for w in os.walk('fail2ban/tests/action_d') for f in w[2]] - }, + } if with_tests else {}, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From 5d0e74d2ab8980356b6a6cb2760b43945e4e34b3 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 16:55:55 +0100 Subject: [PATCH 104/185] man/jail.conf.5: documentation extended (prefregex, etc) closes gh-2820 --- man/jail.conf.5 | 49 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index d7722124..dc226ac2 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -276,6 +276,9 @@ It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "pol .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. .TP +.B prefregex +regex (Python \fBreg\fRular \fBex\fRpression) to parse a common part containing in every message (see \fBprefregex\fR in section FILTER FILES for details). +.TP .B failregex regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes (see \fBfailregex\fR in section FILTER FILES for details). If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below). .TP @@ -432,7 +435,36 @@ These are used to identify failed authentication attempts in log files and to ex Like action files, filter files are ini files. The main section is the [Definition] section. -There are two filter definitions used in the [Definition] section: +There are several standard filter definitions used in the [Definition] section: +.TP +.B prefregex +is the regex (\fBreg\fRular \fBex\fRpression) to parse a common part containing in every message, which is applied after \fBdatepattern\fR found a match, before the search for any \fBfailregex\fR or \fBignoreregex\fR would start. +.br +If this regex doesn't match the process is starting immediately with next message and search for any \fBfailregex\fR does not occur. +.br +If \fBprefregex\fR contains \fI...\fR, the part of message enclosed between this tags will be extracted and herafter used as whole message for search with \fBfailregex\fR or \fBignoreregex\fR. +.IP +For example: +.nf + prefregex = ^%(__prefix_line)s (?:ERROR|FAILURE) .+$ + failregex = ^user not found + ^authentication failed + ^unknown authentication method +.fi +.IP +You can use \fBprefregex\fR in order to: +.RS +.IP +- specify 1 common regex to match some common part present in every messages (do avoid unneeded match in every \fBfailregex\fR if you have more as one); +.IP +- to cut some interesting part of message only (to simplify \fBfailregex\fR) enclosed between tags \fI\fI and \fI\fR; +.IP +- to gather some failure identifier (e. g. some prefix matched by \fI...\fR tag) to identify several messages belonging to same session, where a connect message containing IP followed by failure message(s) that are not contain IP; +this provides a new multi-line parsing method as replacement for old (slow an ugly) multi-line parsing using buffering window (\fImaxlines\fR > 1 and \fI\fR); +.IP +- to ignore some wrong, too long or even unneeded messages (a.k.a. parasite log traffic) which can be also present in journal, before \fBfailregex\fR search would take place. +.RE + .TP .B failregex is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The standard replacement tags can be used as part of the regex: @@ -451,17 +483,18 @@ is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. T \fI\fR - helper regex to match CIDR (simple integer form of net-mask). .IP \fI\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +.PP +\fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message). +.PP +For multiline regexs (parsing with \fImaxlines\fR greater that 1) the tag \fI\fR can be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. +.br +This is an obsolete handling and if the lines contain some common identifier, better would be to use new handling (with tags \fI...\fR). .RE -.TP -For multiline regexs the tag \fI\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. .TP .B ignoreregex is the regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex. - -.PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. @@ -492,7 +525,9 @@ There are several prefixes and words with special meaning that could be specifie \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP -Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. +Similar to actions, filters may have an [Init] section also (optional since v.0.10). All parameters of both sections [Definition] and [Init] can be overridden (redefined or extended) in \fIjail.conf\fR or \fIjail.local\fR (or in related \fIfilter.d/filter-name.local\fR). +Every option supplied in the jail to the filter overwrites the value specified in [Init] section, which in turm would overwrite the value in [Definition] section. +Besides the standard settings of filter both sections can be used to initialize filter-specific options. Filters can also have a section called [INCLUDES]. This is used to read other configuration files. From cc64ef25f68240d30a6653a2fba3cb26d574cd65 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 17:25:41 +0100 Subject: [PATCH 105/185] filter.d/apache-noscript.conf: extended to match "script not found" with error AH02811 (and cgi-bin path segment in script) closes gh-2805 --- config/filter.d/apache-noscript.conf | 4 ++-- fail2ban/tests/files/logs/apache-noscript | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 32991cba..dd9452a9 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -17,9 +17,9 @@ before = apache-common.conf [Definition] -script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl) +script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/) -prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) ).+$ +prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) ).+$ failregex = ^(?:does not exist|not found or unable to stat):