From 412120ac3c3f1c9a3fa8c7676059ab2be0f87c6a Mon Sep 17 00:00:00 2001 From: aresdr Date: Sat, 30 May 2020 15:25:31 -0700 Subject: [PATCH 01/61] Update drupal-auth.conf Small fix for Drupal 8. D8 uses "Login attempt failed from" while D7 uses "Login attempt failed for". The referer part is a must currently, but some requests did not have one and are not failing. --- config/filter.d/drupal-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index b60abe3e..2d4cbe9f 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.+\|\d\|.*\|Login attempt failed for .+\.$ +failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.*\|\d\|.*\|Login attempt failed (?:for|from) .+\.$ ignoreregex = From 27e435a7f5bee112c0b2d79fb0d48898afd16ffd Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 22 Dec 2020 18:17:27 +0100 Subject: [PATCH 02/61] fix cymru test cases --- fail2ban/tests/banmanagertestcase.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index c2c64a35..6f2e1732 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -27,6 +27,7 @@ __license__ = "GPL" import unittest from ..server.banmanager import BanManager +from ..server.ipdns import DNSUtils from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): @@ -137,10 +138,10 @@ class StatusExtendedCymruInfo(unittest.TestCase): """Call before every test case.""" super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() - self.__ban_ip = "93.184.216.34" - self.__asn = "15133" - self.__country = "EU" - self.__rir = "ripencc" + self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() + self.__asn = "36692" + self.__country = "US" + self.__rir = "arin" ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() self.assertTrue(self.__banManager.addBanTicket(ticket)) From 3bb19ecc9d9fe18c3b5ac6a25844d17839543111 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Wed, 23 Dec 2020 12:20:42 +1100 Subject: [PATCH 03/61] docs: fix simple typo, litle -> little There is a small typo in fail2ban/client/fail2banclient.py, fail2ban/server/observer.py. Should read `little` rather than `litle`. --- fail2ban/client/fail2banclient.py | 2 +- fail2ban/server/observer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 6ea18fda..a7053034 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -230,7 +230,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.log(5, ' client phase %s', phase) if not stream: return False - # wait a litle bit for phase "start-ready" before enter active waiting: + # wait a little bit for phase "start-ready" before enter active waiting: if phase is not None: Utils.wait_for(lambda: phase.get('start-ready', None) is not None, 0.5, 0.001) phase['configure'] = (True if stream else False) diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index f5ba20d9..996eec4a 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -232,7 +232,7 @@ class ObserverThread(JailThread): if self._paused: continue else: - ## notify event deleted (shutdown) - just sleep a litle bit (waiting for shutdown events, prevent high cpu usage) + ## notify event deleted (shutdown) - just sleep a little bit (waiting for shutdown events, prevent high cpu usage) time.sleep(ObserverThread.DEFAULT_SLEEP_INTERVAL) ## stop by shutdown and empty queue : if not self.is_full: From ba7daef86c521ceda4ee48b5f221485dd98a13ad Mon Sep 17 00:00:00 2001 From: defanor Date: Thu, 24 Dec 2020 06:55:01 +0300 Subject: [PATCH 04/61] Handle postscreen's PREGREET and HANGUP messages Provoking those seems to be a popular activity among spammers. --- config/filter.d/postfix.conf | 4 +++- fail2ban/tests/files/logs/postfix | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index fb690fb0..69b4ab48 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -37,7 +37,9 @@ mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unava mdpr-more = %(mdpr-normal)s mdre-more = %(mdre-normal)s -mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))) +# Includes some of the log messages described in +# . +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+) mdre-ddos = ^from [^[]*\[\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 6e2dc460..9f74e155 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -151,6 +151,11 @@ Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[1 # failJSON: { "time": "2005-02-18T09:48:04", "match": true , "host": "192.0.2.23" } Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[192.0.2.23] +# failJSON: { "time": "2004-12-23T19:39:13", "match": true , "host": "192.0.2.2" } +Dec 23 19:39:13 xxx postfix/postscreen[21057]: PREGREET 14 after 0.08 from [192.0.2.2]:59415: EHLO ylmf-pc\r\n +# failJSON: { "time": "2004-12-24T00:54:36", "match": true , "host": "192.0.2.3" } +Dec 24 00:54:36 xxx postfix/postscreen[22515]: HANGUP after 16 from [192.0.2.3]:48119 in tests after SMTP handshake + # filterOptions: [{}, {"mode": "ddos"}, {"mode": "aggressive"}] # failJSON: { "match": false, "desc": "don't affect lawful data (sporadical connection aborts within DATA-phase, see gh-1813 for discussion)" } Feb 18 09:50:05 xxx postfix/smtpd[42]: lost connection after DATA from good-host.example.com[192.0.2.10] From 73b39e08947b1858d7574085c3f74318e101e0de Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Dec 2020 21:22:47 +0100 Subject: [PATCH 05/61] filter.d/named-refused.conf: fixes prefix for messages from systemd journal (no mandatory space ahead, because don't have timestamp) closes gh-2899 --- config/filter.d/named-refused.conf | 2 +- fail2ban/tests/files/logs/named-refused | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index 0d5a6275..8a0b1b8c 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -32,7 +32,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re) # hostname daemon_id spaces # this can be optional (for instance if we match named native log files) -__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? +__line_prefix=(?:\s*\S+ %(__daemon_combs_re)s\s+)? prefregex = ^%(__line_prefix)s(?: error:)?\s*client(?: @\S*)? #\S+(?: \([\S.]+\))?: .+\s(?:denied|\(NOTAUTH\))\s*$ diff --git a/fail2ban/tests/files/logs/named-refused b/fail2ban/tests/files/logs/named-refused index 0250276c..c06a4146 100644 --- a/fail2ban/tests/files/logs/named-refused +++ b/fail2ban/tests/files/logs/named-refused @@ -26,3 +26,8 @@ Aug 27 16:58:31 vhost1-ua named[29206]: client 176.9.92.38#42592 (simmarket.com. # failJSON: { "time": "2004-08-27T16:59:00", "match": true , "host": "192.0.2.1", "desc": "new log format, 9.11.0 (#2406)" } Aug 27 16:59:00 host named[28098]: client @0x7f6450002ef0 192.0.2.1#23332 (example.com): bad zone transfer request: 'test.com/IN': non-authoritative zone (NOTAUTH) + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" } +atom named[1806]: client @0x7fb13400eec0 192.0.2.1#61977 (.): query (cache) './ANY/IN' denied From 747d4683221b5584f9663695fb48145689b42ceb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2021 02:42:38 +0100 Subject: [PATCH 06/61] fixes century selector of %ExY and %Exy in datepattern for tests, considering interval from 2005 (alternate now) to now; + better grouping algorithm for resulting century RE --- fail2ban/server/strptime.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 1464a96d..39fc7958 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -36,10 +36,30 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo Thereby respect possible run in the test-cases (alternate date used there) """ cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) exprset = set( cent(now[0].year + i) for i in (-1, distance) ) if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in (-1, distance) ) - return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset) + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) timeRE = TimeRE() From 0a7f5b7b5c7b352e14c13a8058d0a88753e5f48a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:29:18 +0100 Subject: [PATCH 07/61] CI: add python 3.10 to github actions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a1d31df..c5bb5df2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 3c8aa0e6ba4ca46aa730bbb957653e0468365b49 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:34:08 +0100 Subject: [PATCH 08/61] amend with correct yaml notation (float vs string) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5bb5df2..7d69d2e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 097d9ea7fe4cb2e9285203ca56646d0ea003b064 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:36:09 +0100 Subject: [PATCH 09/61] try wildcard (3.10 is still alpha) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d69d2e5..78f40b8a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.*', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From c9907bef0d6fd4d26c9f7a92611cfd03ea434dca Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 Jan 2021 14:37:34 +0100 Subject: [PATCH 10/61] CI: 3.10.0-alpha.4 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78f40b8a..07086636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.*', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-alpha.4', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 2781e7b9d1111005e2bc97c96ddb53f0fbf47f99 Mon Sep 17 00:00:00 2001 From: Michael Haro Date: Fri, 8 Jan 2021 15:01:25 -0800 Subject: [PATCH 11/61] Add blank line so markdown renders correctly --- FILTERS | 1 + 1 file changed, 1 insertion(+) diff --git a/FILTERS b/FILTERS index e114973a..2ed6281d 100644 --- a/FILTERS +++ b/FILTERS @@ -278,6 +278,7 @@ to tune it. fail2ban-regex -D ... will present Debuggex URLs for the regexs and sample log files that you pass into it. In general use when using regex debuggers for generating fail2ban filters: + * use regex from the ./fail2ban-regex output (to ensure all substitutions are done) * replace with (?&.ipv4) From 9df332fdef853c0ae27578d84beda3a1c41deee1 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 11 Jan 2021 15:10:53 +0100 Subject: [PATCH 12/61] filter.d/apache-overflows.conf: extended to match AH00126 error (Invalid URI ...); closes gh-2908 --- config/filter.d/apache-overflows.conf | 2 +- fail2ban/tests/files/logs/apache-overflows | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 02a2ef20..0f54da11 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,7 +8,7 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) +failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) ignoreregex = diff --git a/fail2ban/tests/files/logs/apache-overflows b/fail2ban/tests/files/logs/apache-overflows index 376114c4..4be013eb 100644 --- a/fail2ban/tests/files/logs/apache-overflows +++ b/fail2ban/tests/files/logs/apache-overflows @@ -3,6 +3,8 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# failJSON: { "time": "2010-03-15T16:04:06", "match": true , "host": "192.0.2.1", "desc": "AH00126 failure, gh-2908" } +[Sat Mar 15 16:04:06.105212 2010] [core:error] [pid 17408] [client 192.0.2.1:55280] AH00126: Invalid URI in request GET /static/../../../a/../../../../etc/passwd HTTP/1.1 # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } From 5f3f4d1e2ffcf99384c96c9800d1aa913ccce81b Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 11 Jan 2021 15:23:40 +0100 Subject: [PATCH 13/61] action.d/cloudflare.conf: better IPv6 capability closes gh-2891 --- config/action.d/cloudflare.conf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 361cb177..4af87080 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -44,7 +44,7 @@ actioncheck = #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ - -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + -d '{"mode":"block","configuration":{"target":"","value":""},"notes":"Fail2Ban "}' \ <_cf_api_url> # Option: actionunban @@ -59,7 +59,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ + "<_cf_api_url>?mode=block&configuration_target=&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" @@ -81,3 +81,8 @@ _cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Conten cftoken = cfuser = + +cftarget = ip + +[Init?family=inet6] +cftarget = ip6 From 164105fab12a8d97050eb1930cbc338b66d6e8eb Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 16 Jan 2021 17:10:12 +0100 Subject: [PATCH 14/61] added new parameter `namespace` for systemd backend closes gh-2910 --- fail2ban/server/filtersystemd.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 47fc891e..1b33b115 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -94,6 +94,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # be sure all journal types will be opened if files specified (don't set flags): if 'files' not in args or not len(args['files']): args['flags'] = 4 + + try: + args['namespace'] = kwargs.pop('namespace') + except KeyError: + pass return args From 0f44a3408aa88d32ab2c4583ac6f54ea82220069 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 19 Jan 2021 17:07:03 +0100 Subject: [PATCH 15/61] amend to 747d4683221b5584f9663695fb48145689b42ceb: fail2ban-regex: loosen up date patterns %ExY, %Exy - let accept every year from 19xx up to current century (+3 years) --- fail2ban/client/fail2banregex.py | 11 +++-- fail2ban/server/datedetector.py | 2 +- fail2ban/server/datetemplate.py | 3 +- fail2ban/server/strptime.py | 81 ++++++++++++++++++-------------- 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index e7a4e214..5d5f4a1c 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -39,7 +39,6 @@ import os import shlex import sys import time -import time import urllib from optparse import OptionParser, Option @@ -52,7 +51,7 @@ except ImportError: from ..version import version, normVersion from .filterreader import FilterReader -from ..server.filter import Filter, FileContainer +from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ @@ -269,15 +268,19 @@ class Fail2banRegex(object): self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.timezone: self._filter.setLogTimeZone(opts.timezone) + self._filter.checkFindTime = False + if True: # not opts.out: + MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns) + from ..server.strptime import _updateTimeRE + _updateTimeRE() if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: self._filter.setUseDns(opts.usedns) self._filter.returnRawHost = opts.raw - self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) - self._filter.ignorePending = opts.out + self._filter.ignorePending = bool(opts.out) # callback to increment ignored RE's by index (during process): self._filter.onIgnoreRegex = self._onIgnoreRegex self._backend = 'auto' diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 90a70b0d..ecc9d935 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -35,7 +35,7 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -logLevel = 6 +logLevel = 5 RE_DATE_PREMATCH = re.compile(r"(? 1: - for i in exprset: - if c is None or i[0:-1] == c: - c = i[0:-1] - else: - c = None - break - if not c: - for i in exprset: - if c is None or i[0] == c: - c = i[0] - else: - c = None - break - if c: - return "%s%s" % (c, grp([i[len(c):] for i in exprset])) - return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ - if len(exprset) > 1 else "".join(exprset) - exprset = set( cent(now[0].year + i) for i in (-1, distance) ) - if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) - return grp(sorted(list(exprset))) - timeRE = TimeRE() # %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, @@ -92,11 +61,51 @@ timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" -# more precise year patterns, within same century of last year and -# the next 3 years (for possible long uptime of fail2ban); thereby -# respect possible run in the test-cases (alternate date used there): -timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) -timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) + +def _updateTimeRE(): + def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): + """ Build century regex for last year and the next years (distance). + + Thereby respect possible run in the test-cases (alternate date used there) + """ + cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) + exprset = set( cent(now[0].year + i) for i in (-1, distance) ) + if len(now) > 1 and now[1]: + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) + + # more precise year patterns, within same century of last year and + # the next 3 years (for possible long uptime of fail2ban); thereby + # respect possible run in the test-cases (alternate date used there): + if MyTime.alternateNowTime != 0: + timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) + timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) + else: # accept years: 19xx|2xxx up to current century + timeRE['ExY'] = r"(?P(?:19\d{2}|%s\d))" % _getYearCentRE(cent=(0,3), distance=3, + now=(MyTime.now(), datetime.datetime.fromtimestamp(978393600))) + timeRE['Exy'] = r"(?P\d{2})" + +_updateTimeRE() def getTimePatternRE(): keys = timeRE.keys() From 913c37db80d3f78c37e7df2f18a503baf3f6dabc Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 21 Jan 2021 18:56:25 +0100 Subject: [PATCH 16/61] more fixes and optimizations, better RE's for patterns, allow parse date without time with such a datepattern (assume 00:00:00 then), etc --- fail2ban/server/strptime.py | 22 +++++++++++++++------- fail2ban/tests/datedetectortestcase.py | 3 +++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 83cb3989..69514b20 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -52,15 +52,23 @@ timeRE['z'] = r"(?PZ|UTC|GMT|[+-][01]\d(?::?\d{2})?)" timeRE['ExZ'] = r"(?P%s)" % (TZ_ABBR_RE,) timeRE['Exz'] = r"(?P(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE) +# overwrite default patterns, since they can be non-optimal: +timeRE['d'] = r"(?P[1-2]\d|[0 ]?[1-9]|3[0-1])" +timeRE['m'] = r"(?P0?[1-9]|1[0-2])" +timeRE['Y'] = r"(?P\d{4})" +timeRE['H'] = r"(?P[0-1]?\d|2[0-3])" +timeRE['M'] = r"(?P[0-5]?\d)" +timeRE['S'] = r"(?P[0-5]?\d|6[0-1])" + # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: -timeRE['Exd'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" -timeRE['Exm'] = r"(?P1[0-2]|0[1-9])" -timeRE['ExH'] = r"(?P2[0-3]|[0-1]\d)" -timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" +timeRE['Exd'] = r"(?P[1-2]\d|0[1-9]|3[0-1])" +timeRE['Exm'] = r"(?P0[1-9]|1[0-2])" +timeRE['ExH'] = r"(?P[0-1]\d|2[0-3])" +timeRE['Exk'] = r" ?(?P[0-1]?\d|2[0-3])" timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" -timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" +timeRE['ExS'] = r"(?P[0-5]\d|6[0-1])" def _updateTimeRE(): def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): @@ -197,9 +205,9 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """ now = \ - year = month = day = hour = minute = tzoffset = \ + year = month = day = tzoffset = \ weekday = julian = week_of_year = None - second = fraction = 0 + hour = minute = second = fraction = 0 for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index d6370fc4..b8e8451e 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -551,6 +551,9 @@ class CustomDateFormatsTest(unittest.TestCase): (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), + # no time (only date) in pattern, assume local 00:00:00 for H:M:S : + (1093989600.0, "^%Y-%m-%d**", '2004-09-01'), + (1093996800.0, "^%Y-%m-%d%z**", '2004-09-01Z'), ): logSys.debug('== test: %r', (matched, dp, line)) dd = DateDetector() From 3700a9e5230822d33b2377afed9b6de38676e564 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 26 Jan 2021 20:25:58 +0100 Subject: [PATCH 17/61] invalidate IP/DNS caches by reload, so inter alia would allow to recognize IPv6IsAllowed immediately, previously retarded up to cache max-time (5m); closes gh-2804 --- fail2ban/server/server.py | 9 ++++++++- fail2ban/tests/utils.py | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index b12c8f9f..475fd706 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -33,7 +33,7 @@ import stat import sys from .jails import Jails -from .filter import FileFilter, JournalFilter +from .filter import DNSUtils, FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version @@ -274,6 +274,11 @@ class Server: for name in self.__jails.keys(): self.delJail(name, stop=False, join=True) + def clearCaches(self): + # we need to clear caches, to be able to recognize new IPs/families etc: + DNSUtils.CACHE_nameToIp.clear() + DNSUtils.CACHE_ipToName.clear() + def reloadJails(self, name, opts, begin): if begin: # begin reload: @@ -295,6 +300,8 @@ class Server: if "--restart" in opts: self.stopJail(name) else: + # invalidate caches by reload + self.clearCaches() # first unban all ips (will be not restored after (re)start): if "--unban" in opts: self.setUnbanIP() diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b54581f5..921427db 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -320,6 +320,7 @@ def initTests(opts): # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName + c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) for i in xrange(256): @@ -337,6 +338,7 @@ def initTests(opts): c.set('8.8.4.4', 'dns.google') # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp + c.clear = lambda: logSys.warn('clear CACHE_nameToIp is disabled in test suite') for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), From c75748c5d3b4de1e51318e1db4a5b72519c47c1f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 27 Jan 2021 17:05:37 +0100 Subject: [PATCH 18/61] fail2ban.conf: added new fail2ban configuration option "allowipv6" (default auto), can be used to allow or disallow IPv6 interface in fail2ban immediately by start (e. g. if fail2ban starts before network interfaces). closes gh-2804 --- config/fail2ban.conf | 6 ++++ fail2ban/client/fail2banreader.py | 2 ++ fail2ban/server/ipdns.py | 41 +++++++++++++++++++------- fail2ban/server/server.py | 5 ++++ fail2ban/server/transmitter.py | 5 ++++ fail2ban/tests/clientreadertestcase.py | 1 + fail2ban/tests/servertestcase.py | 15 +++++++++- man/jail.conf.5 | 5 ++++ 8 files changed, 69 insertions(+), 11 deletions(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index f3867839..601402d8 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -55,6 +55,12 @@ socket = /var/run/fail2ban/fail2ban.sock # pidfile = /var/run/fail2ban/fail2ban.pid +# Option: allowipv6 +# Notes.: Allows IPv6 interface: +# Default: auto +# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto +#allowipv6 = auto + # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. # A value of ":memory:" means database is only stored in memory diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 3270b767..1f135cf8 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -53,6 +53,7 @@ class Fail2banReader(ConfigReader): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], + ["string", "allowipv6", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbmaxmatches", None], ["string", "dbpurgeage", "1d"]] @@ -74,6 +75,7 @@ class Fail2banReader(ConfigReader): # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, + "allowipv6": 14, "dbfile":50, "dbmaxmatches":51, "dbpurgeage":51} stream = list() for opt in self.__opts: diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 571ccc4f..5f3e4571 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -169,27 +169,31 @@ class DNSUtils: DNSUtils.CACHE_ipToName.set(key, name) return name + # key find cached own hostnames (this tuple-key cannot be used elsewhere): + _getSelfNames_key = ('self','dns') + @staticmethod def getSelfNames(): """Get own host names of self""" - # try find cached own hostnames (this tuple-key cannot be used elsewhere): - key = ('self','dns') - names = DNSUtils.CACHE_ipToName.get(key) + # try find cached own hostnames: + names = DNSUtils.CACHE_ipToName.get(DNSUtils._getSelfNames_key) # get it using different ways (a set with names of localhost, hostname, fully qualified): if names is None: names = set([ 'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True) ]) - set(['']) # getHostname can return '' # cache and return : - DNSUtils.CACHE_ipToName.set(key, names) + DNSUtils.CACHE_ipToName.set(DNSUtils._getSelfNames_key, names) return names + # key to find cached own IPs (this tuple-key cannot be used elsewhere): + _getSelfIPs_key = ('self','ips') + @staticmethod def getSelfIPs(): """Get own IP addresses of self""" - # try find cached own IPs (this tuple-key cannot be used elsewhere): - key = ('self','ips') - ips = DNSUtils.CACHE_nameToIp.get(key) + # to find cached own IPs: + ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getSelfIPs_key) # get it using different ways (a set with IPs of localhost, hostname, fully qualified): if ips is None: ips = set() @@ -199,13 +203,30 @@ class DNSUtils: except Exception as e: # pragma: no cover logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) # cache and return : - DNSUtils.CACHE_nameToIp.set(key, ips) + DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips) return ips + _IPv6IsAllowed = None + + @staticmethod + def setIPv6IsAllowed(value): + DNSUtils._IPv6IsAllowed = value + logSys.debug("IPv6 is %s", ('on' if value else 'off') if value is not None else 'auto') + return value + + # key to find cached value of IPv6 allowance (this tuple-key cannot be used elsewhere): + _IPv6IsAllowed_key = ('self','ipv6-allowed') + @staticmethod def IPv6IsAllowed(): - # return os.path.exists("/proc/net/if_inet6") || any((':' in ip) for ip in DNSUtils.getSelfIPs()) - return any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + if DNSUtils._IPv6IsAllowed is not None: + return DNSUtils._IPv6IsAllowed + v = DNSUtils.CACHE_nameToIp.get(DNSUtils._IPv6IsAllowed_key) + if v is not None: + return v + v = any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + DNSUtils.CACHE_nameToIp.set(DNSUtils._IPv6IsAllowed_key, v) + return v ## diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 475fd706..bd2c7ad3 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -770,6 +770,11 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + @staticmethod + def setIPv6IsAllowed(value): + value = _as_bool(value) if value != 'auto' else None + return DNSUtils.setIPv6IsAllowed(value) + def setThreadOptions(self, value): for o, v in value.iteritems(): if o == 'stacksize': diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 10cfd163..e40f2f51 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -173,6 +173,11 @@ class Transmitter: return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + elif name == "allowipv6": + value = command[1] + self.__server.setIPv6IsAllowed(value) + if self.__quiet: return + return value #Thread elif name == "thread": value = command[1] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 2cfaff77..54850bca 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -975,6 +975,7 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'syslogsocket', 'auto'], ['set', 'loglevel', "INFO"], ['set', 'logtarget', '/var/log/fail2ban.log'], + ['set', 'allowipv6', 'auto'], ['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'], ['set', 'dbmaxmatches', 10], ['set', 'dbpurgeage', '1d'], diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index eaf1b346..b7b9d802 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -35,7 +35,7 @@ import platform from ..server.failregex import Regex, FailRegex, RegexException from ..server import actions as _actions from ..server.server import Server -from ..server.ipdns import IPAddr +from ..server.ipdns import DNSUtils, IPAddr from ..server.jail import Jail from ..server.jailthread import JailThread from ..server.ticket import BanTicket @@ -175,6 +175,19 @@ class Transmitter(TransmitterBase): def testVersion(self): self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) + def testSetIPv6(self): + try: + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'yes']), (0, 'yes')) + self.assertTrue(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is on"); self.pruneLog() + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'no']), (0, 'no')) + self.assertFalse(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is off"); self.pruneLog() + finally: + # restore back to auto: + self.assertEqual(self.transm.proceed(["set", "allowipv6", "auto"]), (0, "auto")) + self.assertLogged("IPv6 is auto"); self.pruneLog() + def testSleep(self): if not unittest.F2B.fast: t0 = time.time() diff --git a/man/jail.conf.5 b/man/jail.conf.5 index dc226ac2..788fad2b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -151,6 +151,11 @@ PID filename. Default: /var/run/fail2ban/fail2ban.pid .br This is used to store the process ID of the fail2ban server. .TP +.B allowipv6 +option to allow IPv6 interface - auto, yes (on, true, 1) or no (off, false, 0). Default: auto +.br +This value can be used to declare fail2ban whether IPv6 is allowed or not. +.TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 .br From dc4ee5aa47e211bb8ca7417c73b61d4c3dc44a14 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 18 Jan 2021 17:43:06 -0500 Subject: [PATCH 19/61] Add transport to asterisk RE Call rejection messages from Asterisk can have the transport prefixed to the IP address. Signed-off-by: Brian J. Murrell --- config/filter.d/asterisk.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 68472495..e15d7bfe 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -21,7 +21,7 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])?:? [^:]+ prefregex = ^%(__prefix_line)s%(log_prefix)s .+$ failregex = ^Registration from '[^']*' failed for '(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context + ^Call from '[^']*' \((?:(?:TCP|UDP):)?:\d+\) to extension '[^']*' rejected because extension not found in context ^(?:Host )? (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b) ^No registration for peer '[^']*' \(from \)$ ^hacking attempt detected ''$ From 69c96c00c0ed6cdeb2e90a496e91b569c14cef14 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 18 Jan 2021 17:51:38 -0500 Subject: [PATCH 20/61] Log entries for updated failregex Add a sample failregex. Signed-off-by: Brian J. Murrell --- fail2ban/tests/files/logs/asterisk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 76ec40b2..0c4f81bf 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -19,6 +19,8 @@ [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. +# failJSON: { "time": "Jan 18 17:39:50", "match": true , "host": "1.2.3.4" } +[Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'. # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found # failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" } From 7f185a828e6573669535b152fb945a366270d809 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 19 Jan 2021 08:42:05 -0500 Subject: [PATCH 21/61] Update date in failJSON The date format in failJSON is specific, so convert the date to use that format. --- fail2ban/tests/files/logs/asterisk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 0c4f81bf..ab31fa6f 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -19,7 +19,7 @@ [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. -# failJSON: { "time": "Jan 18 17:39:50", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-01-18T17:39:50", "match": true , "host": "1.2.3.4" } [Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'. # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found From 366c64cb9da3c5d833eefc83f5bb200713db29b5 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Feb 2021 14:45:30 +0100 Subject: [PATCH 22/61] extractOptions: ensure options are parsed completely - avoids unexpected skip or truncate of parameters, produces more verbose error message in case of incorrect syntax; added more tests covering several cases WARN: potential incompatibility (since it doesn't silently ignore wrong syntax anymore) --- fail2ban/client/fail2banregex.py | 40 +++++++++++++++---------- fail2ban/client/jailreader.py | 15 +++++----- fail2ban/helpers.py | 12 ++++++-- fail2ban/tests/clientreadertestcase.py | 17 ++++++----- fail2ban/tests/fail2banregextestcase.py | 6 ++++ 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 5d5f4a1c..90e178f9 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -35,6 +35,7 @@ __license__ = "GPL" import getopt import logging +import re import os import shlex import sys @@ -329,26 +330,33 @@ class Fail2banRegex(object): regex = regextype + 'regex' # try to check - we've case filter?[options...]?: basedir = self._opts.config + fltName = value fltFile = None fltOpt = {} if regextype == 'fail': - fltName, fltOpt = extractOptions(value) - if fltName is not None: - if "." in fltName[~5:]: - tryNames = (fltName,) - else: - tryNames = (fltName, fltName + '.conf', fltName + '.local') - for fltFile in tryNames: - if not "/" in fltFile: - if os.path.basename(basedir) == 'filter.d': - fltFile = os.path.join(basedir, fltFile) - else: - fltFile = os.path.join(basedir, 'filter.d', fltFile) + if re.search(r'^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): + try: + fltName, fltOpt = extractOptions(value) + if "." in fltName[~5:]: + tryNames = (fltName,) else: - basedir = os.path.dirname(fltFile) - if os.path.isfile(fltFile): - break - fltFile = None + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + except Exception as e: + output("ERROR: Wrong filter name or options: %s" % (str(e),)) + output(" while parsing: %s" % (value,)) + if self._verbose: raise(e) + return False # if it is filter file: if fltFile is not None: if (basedir == self._opts.config diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 1d7db0dc..0a27c644 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -133,9 +133,10 @@ class JailReader(ConfigReader): # Read filter flt = self.__opts["filter"] if flt: - filterName, filterOpt = extractOptions(flt) - if not filterName: - raise JailDefError("Invalid filter definition %r" % flt) + try: + filterName, filterOpt = extractOptions(flt) + except ValueError as e: + raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) @@ -167,10 +168,10 @@ class JailReader(ConfigReader): if not act: # skip empty actions continue # join with previous line if needed (consider possible new-line): - actName, actOpt = extractOptions(act) - prevln = '' - if not actName: - raise JailDefError("Invalid action definition %r" % act) + try: + actName, actOpt = extractOptions(act) + except ValueError as e: + raise JailDefError("Invalid action definition %r: %s" % (act, e)) if actName.endswith(".py"): self.__actions.append([ "set", diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index c45be849..5c1750a6 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -371,7 +371,7 @@ OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL) # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` OPTION_EXTRACT_CRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) + r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P.+))|,?\s*$|(?P.+)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) @@ -379,13 +379,19 @@ OPTION_SPLIT_CRE = re.compile( def extractOptions(option): match = OPTION_CRE.match(option) if not match: - # TODO proper error handling - return None, None + raise ValueError("unexpected option syntax") option_name, optstr = match.groups() option_opts = dict() if optstr: for optmatch in OPTION_EXTRACT_CRE.finditer(optstr): + if optmatch.group("wrngA"): + raise ValueError("unexpected syntax at %d after option %r: %s" % ( + optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25])) + if optmatch.group("wrngB"): + raise ValueError("expected option, wrong syntax at %d: %s" % ( + optmatch.start("wrngB"), optmatch.group("wrngB")[0:25])) opt = optmatch.group(1) + if not opt: continue value = [ val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 54850bca..e92edd48 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -381,13 +381,16 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]")) self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is")) - self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']")) - #print(self.getLog()) - #self.assertLogged("Invalid argument ['s'] in ''s''") - self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']")) + self.assertEqual(('mail', {'a': 'b'}), extractOptions("mail[a=b, ]")) - #self.assertRaises(ValueError, extractOptions ,'mail-how[') + self.assertRaises(ValueError, extractOptions ,'mail-how[') + + self.assertRaises(ValueError, extractOptions, """mail[a="test with interim (wrong) "" quotes"]""") + self.assertRaises(ValueError, extractOptions, """mail[a='test with interim (wrong) '' quotes']""") + self.assertRaises(ValueError, extractOptions, """mail[a='x, y, z', b=x, y, z]""") + + self.assertRaises(ValueError, extractOptions, """mail['s']""") # Empty option option = "abc[]" @@ -752,9 +755,9 @@ class JailsReaderTest(LogCaptureTestCase): ['add', 'tz_correct', 'auto'], ['start', 'tz_correct'], ['config-error', - "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], + "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"], ['config-error', - "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"], + "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"], ['config-error', "Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"], ['config-error', diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 4d878a24..85fe4f15 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -141,6 +141,12 @@ class Fail2banRegexTest(LogCaptureTestCase): )) self.assertLogged("Unable to compile regular expression") + def testWrongFilterOptions(self): + self.assertFalse(_test_exec( + "test", "flt[a='x,y,z',b=z,y,x]" + )) + self.assertLogged("Wrong filter name or options", "wrong syntax at 14: y,x", all=True) + def testDirectFound(self): self.assertTrue(_test_exec( "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", From d678440658541a3c02c8d57695bd363f72147724 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 11 Feb 2021 18:32:32 +0100 Subject: [PATCH 23/61] more precise RE (avoids weakness with catch-all's and is injection safe) --- config/filter.d/drupal-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index 2d4cbe9f..2404cc6d 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|\|.+\|.*\|\d\|.*\|Login attempt failed (?:for|from) .+\.$ +failregex = ^%(__prefix_line)s(?:https?:\/\/)[^|]+\|[^|]+\|[^|]+\|\|(?:[^|]*\|)*Login attempt failed (?:for|from) [^|]+\.$ ignoreregex = From f4f92aa72d4124fbdff615e14fc5bfb8d66d7b0a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 11 Feb 2021 18:56:53 +0100 Subject: [PATCH 24/61] more tests covering different cases, injections attempt etc --- fail2ban/tests/files/logs/drupal-auth | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/drupal-auth b/fail2ban/tests/files/logs/drupal-auth index 5e7194d9..4d063e55 100644 --- a/fail2ban/tests/files/logs/drupal-auth +++ b/fail2ban/tests/files/logs/drupal-auth @@ -3,5 +3,15 @@ Apr 26 13:15:25 webserver example.com: https://example.com|1430068525|user|1.2.3 # failJSON: { "time": "2005-04-26T13:15:25", "match": true , "host": "1.2.3.4" } Apr 26 13:15:25 webserver example.com: https://example.com/subdir|1430068525|user|1.2.3.4|https://example.com/subdir/user|https://example.com/subdir/user|0||Login attempt failed for drupaladmin. -# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4" } +# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4", "user": "drupaladmin" } Apr 26 13:19:08 webserver example.com: https://example.com|1430068748|user|1.2.3.4|https://example.com/user|https://example.com/user|1||Session opened for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:00", "match": false, "desc": "attempt to inject on URI (pipe, login failed for), not a failure, gh-2742" } +Apr 26 13:20:00 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed for tester|2||Session revisited for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:01", "match": true , "host": "192.0.2.7", "user": "Jack Sparrow", "desc": "log-format change - for -> from, user name with space, gh-2742" } +Apr 26 13:20:01 mweb drupal_site[24864]: https://www.example.com|1613058599|user|192.0.2.7|https://www.example.com/en/user/login|https://www.example.com/en/user/login|0||Login attempt failed from Jack Sparrow. +# failJSON: { "time": "2005-04-26T13:20:02", "match": true , "host": "192.0.2.4", "desc": "attempt to inject on URI (pipe), login failed, gh-2742" } +Apr 26 13:20:02 host drupal-site: https://example.com|1613063581|user|192.0.2.4|https://example.com/user/login?test=%7C&test2=%7C|https://example.com/user/login?test=|&test2=||0||Login attempt failed from 192.0.2.4. +# failJSON: { "time": "2005-04-26T13:20:03", "match": false, "desc": "attempt to inject on URI (pipe, login failed from), not a failure, gh-2742" } +Apr 26 13:20:03 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed from 1.2.3.4|2||Session revisited for drupaladmin. From 8ae9208454e426aa87b96ba5df26036c4ae5cefd Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 16:44:27 +0100 Subject: [PATCH 25/61] try to provide coverage for 3.10-alpha.5 (#2931) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a1d31df..262448c2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-alpha.5', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: From 2b6bb2c1bed8f7009631e8f8c306fa3160324a49 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:19:24 +0100 Subject: [PATCH 26/61] follow bpo-37324: :ref:`collections-abstract-base-classes` moved to the :mod:`collections.abc` module (since 3.10-alpha.5 `MutableMapping` is missing in collections module) --- fail2ban/server/action.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 3bc48fe0..f0f1e6f5 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -30,7 +30,10 @@ import tempfile import threading import time from abc import ABCMeta -from collections import MutableMapping +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from .failregex import mapTag2Opt from .ipdns import DNSUtils From 42dee38ad2ac5c3f23bdf297d824022923270dd9 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:25:45 +0100 Subject: [PATCH 27/61] amend for `Mapping` --- fail2ban/server/actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index b7b95b44..897d907c 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -28,7 +28,10 @@ import logging import os import sys import time -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping try: from collections import OrderedDict except ImportError: From 9f1d1f4fbd0804695a976beb191f2c49a2739834 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 8 Feb 2021 17:35:59 +0100 Subject: [PATCH 28/61] amend for `Mapping` (jails) --- fail2ban/server/jails.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py index 972a8c4b..27e12ddf 100644 --- a/fail2ban/server/jails.py +++ b/fail2ban/server/jails.py @@ -22,7 +22,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" from threading import Lock -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from ..exceptions import DuplicateJailException, UnknownJailException from .jail import Jail From abc5a4e062417139893ba1f402e09e96f8887479 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 Feb 2021 19:02:22 +0100 Subject: [PATCH 29/61] ChangeLog (#2742) --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 6c530871..e5e7b485 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition different from 0) in case of unsane environment. ### Fixes +* `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) ### New Features and Enhancements * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair From 55d7d9e214f72bbe4f39a2d17aa004d80bfc7299 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Feb 2021 18:39:58 +0100 Subject: [PATCH 30/61] *WiP* try to solve RC on jails with too many failures without ban, gh-2945 ... --- fail2ban/server/failmanager.py | 5 +++-- fail2ban/server/filter.py | 34 +++++++++++++++++++++++------- fail2ban/server/filtergamin.py | 16 +++----------- fail2ban/server/filterpoll.py | 10 ++------- fail2ban/server/filterpyinotify.py | 15 ++++++------- fail2ban/server/filtersystemd.py | 13 ++++++------ fail2ban/tests/filtertestcase.py | 14 ++++++------ 7 files changed, 53 insertions(+), 54 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 4173a233..64576dbd 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -124,9 +124,10 @@ class FailManager: return len(self.__failList) def cleanup(self, time): + time -= self.__maxTime with self.__lock: todelete = [fid for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime <= time] + if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: self.__failList = dict() @@ -140,7 +141,7 @@ class FailManager: else: # create new dictionary without items to be deleted: self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime > time) + if item.getTime() > time) self.__bgSvc.service() def delFailure(self, fid): diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 4e947d27..0f4e9e5b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -93,6 +93,8 @@ class Filter(JailThread): ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## Next service (cleanup) time + self.__nextSvcTime = -(1<<63) ## if set, treat log lines without explicit time zone to be in this time zone self.__logtimezone = None ## Default or preferred encoding (to decode bytes from file or journal): @@ -114,10 +116,10 @@ class Filter(JailThread): self.checkFindTime = True ## shows that filter is in operation mode (processing new messages): self.inOperation = True - ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): - self.banASAP = True ## Ticks counter self.ticks = 0 + ## Processed lines counter + self.procLines = 0 ## Thread name: self.name="f2b/f."+self.jailName @@ -441,12 +443,23 @@ class Filter(JailThread): def performBan(self, ip=None): """Performs a ban for IPs (or given ip) that are reached maxretry of the jail.""" - try: # pragma: no branch - exception is the only way out - while True: + while True: + try: ticket = self.failManager.toBan(ip) - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) + except FailManagerEmpty: + break + self.jail.putFailTicket(ticket) + if ip: break + self.performSvc() + + def performSvc(self, force=False): + """Performs a service tasks (clean failure list).""" + tm = MyTime.time() + # avoid too early clean up: + if force or tm >= self.__nextSvcTime: + self.__nextSvcTime = tm + 5 + # clean up failure list: + self.failManager.cleanup(tm) def addAttempt(self, ip, *matches): """Generate a failed attempt for ip""" @@ -694,8 +707,12 @@ class Filter(JailThread): attempts = self.failManager.addFailure(tick) # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, # we can speedup ban, so do it as soon as possible: - if self.banASAP and attempts >= self.failManager.getMaxRetry(): + if attempts >= self.failManager.getMaxRetry(): self.performBan(ip) + self.procLines += 1 + # every 100 lines check need to perform service tasks: + if self.procLines % 100 == 0: + self.performSvc() # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 @@ -1064,6 +1081,7 @@ class FileFilter(Filter): # is created and is added to the FailManager. def getFailures(self, filename, inOperation=None): + if self.idle: return False log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 078246de..c5373445 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -55,7 +55,6 @@ class FilterGamin(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Gamin monitor self.monitor = gamin.WatchMonitor() fd = self.monitor.get_fd() @@ -67,21 +66,9 @@ class FilterGamin(FileFilter): logSys.log(4, "Got event: " + repr(event) + " for " + path) if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists): logSys.debug("File changed: " + path) - self.__modified = True self.ticks += 1 - self._process_file(path) - - def _process_file(self, path): - """Process a given file - - TODO -- RF: - this is a common logic and must be shared/provided by FileFilter - """ self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False ## # Add a log file path @@ -128,6 +115,9 @@ class FilterGamin(FileFilter): Utils.wait_for(lambda: not self.active or self._handleEvents(), self.sleeptime) self.ticks += 1 + if self.ticks % 10 == 0: + self.performSvc() + logSys.debug("[%s] filter terminated", self.jailName) return True diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 7bbdfc5c..7ee00540 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -27,9 +27,7 @@ __license__ = "GPL" import os import time -from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime from .utils import Utils from ..helpers import getLogger, logging @@ -55,7 +53,6 @@ class FilterPoll(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() @@ -115,13 +112,10 @@ class FilterPoll(FileFilter): break for filename in modlst: self.getFailures(filename) - self.__modified = True self.ticks += 1 - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 9796e26f..d62348a2 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -75,7 +75,6 @@ class FilterPyinotify(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__notifier = None @@ -140,9 +139,6 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -352,9 +348,14 @@ class FilterPyinotify(FileFilter): if not self.active: break self.__notifier.read_events() + self.ticks += 1 + # check pending files/dirs (logrotate ready): - if not self.idle: - self._checkPending() + if self.idle: + continue + self._checkPending() + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -364,8 +365,6 @@ class FilterPyinotify(FileFilter): # incr common error counter: self.commonError() - self.ticks += 1 - logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 1b33b115..925109d1 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -322,13 +322,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover break else: break - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = 0 - # update position in log (time and iso string): - if self.jail.database is not None: - self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__modified = 0 + if self.ticks % 10 == 0: + self.performSvc() + # update position in log (time and iso string): + if self.jail.database is not None: + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2dac91d1..15882ea0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -800,7 +800,6 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -952,15 +951,18 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') + print('=========='*10) self.filter.getFailures(self.name) + print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... + print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) - _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) + #_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1018,7 +1020,6 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1277,14 +1278,14 @@ def get_monitor_failures_testcase(Filter_): # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) # we should detect the failures - self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 12) + self.assertEqual(self.filter.failManager.getFailTotal(), 9) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1316,7 +1317,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1570,7 +1570,6 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1771,7 +1770,6 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) - filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures From e353fb802442309d0b6fbfe86cf6d0ab286c6626 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 02:46:44 +0100 Subject: [PATCH 31/61] fixed test cases (ban ASAP also followed in test suite now, so failure reached maxretry causes immediate ban now) --- fail2ban/tests/filtertestcase.py | 120 +++++++++++++++++-------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 15882ea0..fe37ea29 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -164,18 +164,25 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): # get fail ticket from jail found.append(_ticket_tuple(filter_.getFailTicket())) else: - # when we are testing without jails - # wait for failures (up to max time) - Utils.wait_for( - lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), - _maxWaitTime(10)) - # get fail ticket(s) from filter - while tickcount: - try: - found.append(_ticket_tuple(filter_.failManager.toBan())) - except FailManagerEmpty: - break - tickcount -= 1 + # when we are testing without jails wait for failures (up to max time) + if filter_.jail: + while True: + t = filter_.jail.getFailTicket() + if not t: break + found.append(_ticket_tuple(t)) + if found: + tickcount -= len(found) + if tickcount > 0: + Utils.wait_for( + lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), + _maxWaitTime(10)) + # get fail ticket(s) from filter + while tickcount: + try: + found.append(_ticket_tuple(filter_.failManager.toBan())) + except FailManagerEmpty: + break + tickcount -= 1 if not isinstance(output[0], (tuple,list)): utest.assertEqual(len(found), 1) @@ -951,14 +958,11 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') - print('=========='*10) self.filter.getFailures(self.name) - print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... - print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) @@ -1112,12 +1116,13 @@ def get_monitor_failures_testcase(Filter_): skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def _wait4failures(self, count=2): + def _wait4failures(self, count=2, waitEmpty=True): # Poll might need more time - self.assertTrue(self.isEmpty(_maxWaitTime(5)), - "Queue must be empty but it is not: %s." - % (', '.join([str(x) for x in self.jail.queue]))) - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + if waitEmpty: + self.assertTrue(self.isEmpty(_maxWaitTime(5)), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), count) @@ -1283,9 +1288,9 @@ def get_monitor_failures_testcase(Filter_): # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... - self.assert_correct_last_attempt(GetFailures.FAILURES_01) + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 9) + self._wait4failures(12, False) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1640,6 +1645,7 @@ class GetFailures(LogCaptureTestCase): [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) + self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) self.filter.addFailRegex(r"Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1648,6 +1654,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03(self): output = ('203.162.223.135', 6, 1124013600.0) + self.filter.setMaxRetry(6) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) @@ -1656,6 +1663,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_InOperation(self): output = ('203.162.223.135', 9, 1124013600.0) + self.filter.setMaxRetry(9) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) @@ -1673,7 +1681,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... output = ('203.162.223.135', 2, 1124013600.0) - self.filter.setMaxRetry(1) + self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1683,10 +1691,12 @@ class GetFailures(LogCaptureTestCase): def testGetFailures04(self): # because of not exact time in testcase04.log (no year), we should always use our test time: self.assertEqual(MyTime.time(), 1124013600) - # should find exact 4 failures for *.186 and 2 failures for *.185 - output = (('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 2, 1124013598.0)) - + # should find exact 4 failures for *.186 and 2 failures for *.185, but maxretry is 2, so 3 tickets: + output = ( + ('212.41.96.186', 2, 1124013480.0), + ('212.41.96.186', 2, 1124013600.0), + ('212.41.96.185', 2, 1124013598.0) + ) # speedup search using exact date pattern: self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', @@ -1743,9 +1753,11 @@ class GetFailures(LogCaptureTestCase): unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ( - ('93.184.216.34', 2, 1124013539.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ('93.184.216.34', 1, 1124013299.0, + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ), + ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] @@ -1779,8 +1791,11 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, filter_, output) def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = [ + ('141.3.81.106', 8, 1124013541.0) + ] + self.filter.setMaxRetry(8) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex(r"Failed .* from ") self.filter.addFailRegex(r"Accepted .* from ") @@ -1798,26 +1813,25 @@ class GetFailures(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + + _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1826,14 +1840,17 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + ("192.0.43.15", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1842,14 +1859,9 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) class DNSUtilsTests(unittest.TestCase): From 92a224217496fe3114fc7ee9f80708c00804ec03 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 15:54:48 +0100 Subject: [PATCH 32/61] amend fixing journal tests (systemd backend only) --- fail2ban/tests/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fe37ea29..b9b7e8aa 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1517,7 +1517,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", "TEST_UUID=%s" % self.test_uuid]) - self.assert_correct_ban("193.168.0.128", 4) + self.assert_correct_ban("193.168.0.128", 3) _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures @@ -1531,7 +1531,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=15, n=4) self.waitForTicks(1) self.assertTrue(self.isFilled(10)) - self.assert_correct_ban("87.142.124.10", 4) + self.assert_correct_ban("87.142.124.10", 3) # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", From 6f4b6ec8ccdb68c75aec8225d8fa2b03ed19f320 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2021 13:05:04 +0100 Subject: [PATCH 33/61] action.d/badips.* removed (badips.com is no longer active, gh-2889) --- MANIFEST | 3 - config/action.d/badips.conf | 19 -- config/action.d/badips.py | 391 ------------------------- config/jail.conf | 14 - fail2ban/tests/action_d/test_badips.py | 157 ---------- fail2ban/tests/clientreadertestcase.py | 10 +- 6 files changed, 3 insertions(+), 591 deletions(-) delete mode 100644 config/action.d/badips.conf delete mode 100644 config/action.d/badips.py delete mode 100644 fail2ban/tests/action_d/test_badips.py diff --git a/MANIFEST b/MANIFEST index 50f308db..efe87085 100644 --- a/MANIFEST +++ b/MANIFEST @@ -5,8 +5,6 @@ bin/fail2ban-testcases ChangeLog config/action.d/abuseipdb.conf config/action.d/apf.conf -config/action.d/badips.conf -config/action.d/badips.py config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf @@ -219,7 +217,6 @@ fail2ban/setup.py fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py -fail2ban/tests/action_d/test_badips.py fail2ban/tests/action_d/test_smtp.py fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf deleted file mode 100644 index 6f9513f6..00000000 --- a/config/action.d/badips.conf +++ /dev/null @@ -1,19 +0,0 @@ -# Fail2ban reporting to badips.com -# -# Note: This reports an IP only and does not actually ban traffic. Use -# another action in the same jail if you want bans to occur. -# -# Set the category to the appropriate value before use. -# -# To get see register and optional key to get personalised graphs see: -# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key - -[Definition] - -actionban = curl --fail --user-agent "" http://www.badips.com/add// - -[Init] - -# Option: category -# Notes.: Values are from the list here: http://www.badips.com/get/categories -category = diff --git a/config/action.d/badips.py b/config/action.d/badips.py deleted file mode 100644 index 805120e9..00000000 --- a/config/action.d/badips.py +++ /dev/null @@ -1,391 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import sys -if sys.version_info < (2, 7): # pragma: no cover - raise ImportError("badips.py action requires Python >= 2.7") -import json -import threading -import logging -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.request import Request, urlopen - from urllib.parse import urlencode - from urllib.error import HTTPError -else: # pragma: 3.x no cover - from urllib2 import Request, urlopen, HTTPError - from urllib import urlencode - -from fail2ban.server.actions import Actions, ActionBase, BanTicket -from fail2ban.helpers import splitwords, str2LogLevel - - - -class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable - """Fail2Ban action which reports bans to badips.com, and also - blacklist bad IPs listed on badips.com by using another action's - ban method. - - Parameters - ---------- - jail : Jail - The jail which the action belongs to. - name : str - Name assigned to the action. - category : str - Valid badips.com category for reporting failures. - score : int, optional - Minimum score for bad IPs. Default 3. - age : str, optional - Age of last report for bad IPs, per badips.com syntax. - Default "24h" (24 hours) - banaction : str, optional - Name of banaction to use for blacklisting bad IPs. If `None`, - no blacklist of IPs will take place. - Default `None`. - bancategory : str, optional - Name of category to use for blacklisting, which can differ - from category used for reporting. e.g. may want to report - "postfix", but want to use whole "mail" category for blacklist. - Default `category`. - bankey : str, optional - Key issued by badips.com to retrieve personal list - of blacklist IPs. - updateperiod : int, optional - Time in seconds between updating bad IPs blacklist. - Default 900 (15 minutes) - loglevel : int/str, optional - Log level of the message when an IP is (un)banned. - Default `DEBUG`. - Can be also supplied as two-value list (comma- or space separated) to - provide level of the summary message when a group of IPs is (un)banned. - Example `DEBUG,INFO`. - agent : str, optional - User agent transmitted to server. - Default `Fail2Ban/ver.` - - Raises - ------ - ValueError - If invalid `category`, `score`, `banaction` or `updateperiod`. - """ - - TIMEOUT = 10 - _badips = "https://www.badips.com" - def _Request(self, url, **argv): - return Request(url, headers={'User-Agent': self.agent}, **argv) - - def __init__(self, jail, name, category, score=3, age="24h", - banaction=None, bancategory=None, bankey=None, updateperiod=900, - loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): - super(BadIPsAction, self).__init__(jail, name) - - self.timeout = timeout - self.agent = agent - self.category = category - self.score = score - self.age = age - self.banaction = banaction - self.bancategory = bancategory or category - self.bankey = bankey - loglevel = splitwords(loglevel) - self.sumloglevel = str2LogLevel(loglevel[-1]) - self.loglevel = str2LogLevel(loglevel[0]) - self.updateperiod = updateperiod - - self._bannedips = set() - # Used later for threading.Timer for updating badips - self._timer = None - - @staticmethod - def isAvailable(timeout=1): - try: - response = urlopen(Request("/".join([BadIPsAction._badips]), - headers={'User-Agent': "Fail2Ban"}), timeout=timeout) - return True, '' - except Exception as e: # pragma: no cover - return False, e - - def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) - messages = {} - try: - messages = json.loads(response.read().decode('utf-8')) - except: - pass - self._logSys.error( - "%s. badips.com response: '%s'", what, - messages.get('err', 'Unknown')) - - def getCategories(self, incParents=False): - """Get badips.com categories. - - Returns - ------- - set - Set of categories. - - Raises - ------ - HTTPError - Any issues with badips.com request. - ValueError - If badips.com response didn't contain necessary information - """ - try: - response = urlopen( - self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch categories") - raise - else: - response_json = json.loads(response.read().decode('utf-8')) - if not 'categories' in response_json: - err = "badips.com response lacked categories specification. Response was: %s" \ - % (response_json,) - self._logSys.error(err) - raise ValueError(err) - categories = response_json['categories'] - categories_names = set( - value['Name'] for value in categories) - if incParents: - categories_names.update(set( - value['Parent'] for value in categories - if "Parent" in value)) - return categories_names - - def getList(self, category, score, age, key=None): - """Get badips.com list of bad IPs. - - Parameters - ---------- - category : str - Valid badips.com category. - score : int - Minimum score for bad IPs. - age : str - Age of last report for bad IPs, per badips.com syntax. - key : str, optional - Key issued by badips.com to fetch IPs reported with the - associated key. - - Returns - ------- - set - Set of bad IPs. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "?".join([ - "/".join([self._badips, "get", "list", category, str(score)]), - urlencode({'age': age})]) - if key: - url = "&".join([url, urlencode({'key': key})]) - self._logSys.debug('badips.com: get list, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch bad IP list") - raise - else: - return set(response.read().decode('utf-8').split()) - - @property - def category(self): - """badips.com category for reporting IPs. - """ - return self._category - - @category.setter - def category(self, category): - if category not in self.getCategories(): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - category) - raise ValueError("Invalid category: %s" % category) - self._category = category - - @property - def bancategory(self): - """badips.com bancategory for fetching IPs. - """ - return self._bancategory - - @bancategory.setter - def bancategory(self, bancategory): - if bancategory != "any" and bancategory not in self.getCategories(incParents=True): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - bancategory) - raise ValueError("Invalid bancategory: %s" % bancategory) - self._bancategory = bancategory - - @property - def score(self): - """badips.com minimum score for fetching IPs. - """ - return self._score - - @score.setter - def score(self, score): - score = int(score) - if 0 <= score <= 5: - self._score = score - else: - raise ValueError("Score must be 0-5") - - @property - def banaction(self): - """Jail action to use for banning/unbanning. - """ - return self._banaction - - @banaction.setter - def banaction(self, banaction): - if banaction is not None and banaction not in self._jail.actions: - self._logSys.error("Action name '%s' not in jail '%s'", - banaction, self._jail.name) - raise ValueError("Invalid banaction") - self._banaction = banaction - - @property - def updateperiod(self): - """Period in seconds between banned bad IPs will be updated. - """ - return self._updateperiod - - @updateperiod.setter - def updateperiod(self, updateperiod): - updateperiod = int(updateperiod) - if updateperiod > 0: - self._updateperiod = updateperiod - else: - raise ValueError("Update period must be integer greater than 0") - - def _banIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].ban(ai) - except Exception as e: - self._logSys.error( - "Error banning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._bannedips.add(ip) - self._logSys.log(self.loglevel, - "Banned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - - def _unbanIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].unban(ai) - except Exception as e: - self._logSys.error( - "Error unbanning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._logSys.log(self.loglevel, - "Unbanned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - finally: - self._bannedips.remove(ip) - - def start(self): - """If `banaction` set, blacklists bad IPs. - """ - if self.banaction is not None: - self.update() - - def update(self): - """If `banaction` set, updates blacklisted IPs. - - Queries badips.com for list of bad IPs, removing IPs from the - blacklist if no longer present, and adds new bad IPs to the - blacklist. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - - try: - ips = self.getList( - self.bancategory, self.score, self.age, self.bankey) - # Remove old IPs no longer listed - s = self._bannedips - ips - m = len(s) - self._unbanIPs(s) - # Add new IPs which are now listed - s = ips - self._bannedips - p = len(s) - self._banIPs(s) - if m != 0 or p != 0: - self._logSys.log(self.sumloglevel, - "Updated IPs for jail '%s' (-%d/+%d)", - self._jail.name, m, p) - self._logSys.debug( - "Next update for jail '%' in %i seconds", - self._jail.name, self.updateperiod) - finally: - self._timer = threading.Timer(self.updateperiod, self.update) - self._timer.start() - - def stop(self): - """If `banaction` set, clears blacklisted IPs. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - self._unbanIPs(self._bannedips.copy()) - - def ban(self, aInfo): - """Reports banned IP to badips.com. - - Parameters - ---------- - aInfo : dict - Dictionary which includes information in relation to - the ban. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - self._logSys.debug('badips.com: ban, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to ban") - raise - else: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.debug( - "Response from badips.com report: '%s'", - messages['suc']) - -Action = BadIPsAction diff --git a/config/jail.conf b/config/jail.conf index ddbcf61e..be035112 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -204,20 +204,6 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. -# -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] - # Report ban via abuseipdb.com. # # See action.d/abuseipdb.conf for usage example and details. diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py deleted file mode 100644 index 013c0fdb..00000000 --- a/fail2ban/tests/action_d/test_badips.py +++ /dev/null @@ -1,157 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import unittest -import sys -from functools import wraps -from socket import timeout -from ssl import SSLError - -from ..actiontestcase import CallingMap -from ..dummyjail import DummyJail -from ..servertestcase import IPAddr -from ..utils import LogCaptureTestCase, CONFIG_DIR - -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.error import HTTPError, URLError -else: # pragma: 3.x no cover - from urllib2 import HTTPError, URLError - -def skip_if_not_available(f): - """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". - """ - @wraps(f) - def wrapper(self, *args): - try: - return f(self, *args) - except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues - if not isinstance(e, timeout) and 'timed out' not in str(e): - if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: - raise - raise unittest.SkipTest('Skip test because of %s' % e) - return wrapper - -if sys.version_info >= (2,7): # pragma: no cover - may be unavailable - class BadIPsActionTest(LogCaptureTestCase): - - available = True, None - pythonModule = None - modAction = None - - @skip_if_not_available - def setUp(self): - """Call before every test case.""" - super(BadIPsActionTest, self).setUp() - unittest.F2B.SkipIfNoNetwork() - - self.jail = DummyJail() - - self.jail.actions.add("test") - - pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py") - - # check availability (once if not alive, used shorter timeout as in test cases): - if BadIPsActionTest.available[0]: - if not BadIPsActionTest.modAction: - if not BadIPsActionTest.pythonModule: - BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action - self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) - if not BadIPsActionTest.available[0]: - raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) - - self.jail.actions.add("badips", pythonModuleName, initOpts={ - 'category': "ssh", - 'banaction': "test", - 'age': "2w", - 'score': 5, - #'key': "fail2ban-test-suite", - #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 60), - }) - self.action = self.jail.actions["badips"] - - def tearDown(self): - """Call after every test case.""" - # Must cancel timer! - if self.action._timer: - self.action._timer.cancel() - super(BadIPsActionTest, self).tearDown() - - @skip_if_not_available - def testCategory(self): - categories = self.action.getCategories() - self.assertIn("ssh", categories) - self.assertTrue(len(categories) >= 10) - - self.assertRaises( - ValueError, setattr, self.action, "category", - "invalid-category") - - # Not valid for reporting category... - self.assertRaises( - ValueError, setattr, self.action, "category", "mail") - # but valid for blacklisting. - self.action.bancategory = "mail" - - @skip_if_not_available - def testScore(self): - self.assertRaises(ValueError, setattr, self.action, "score", -5) - self.action.score = 3 - self.action.score = "3" - - @skip_if_not_available - def testBanaction(self): - self.assertRaises( - ValueError, setattr, self.action, "banaction", - "invalid-action") - self.action.banaction = "test" - - @skip_if_not_available - def testUpdateperiod(self): - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", -50) - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", 0) - self.action.updateperiod = 900 - self.action.updateperiod = "900" - - @skip_if_not_available - def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - - @skip_if_not_available - def testBanIP(self): - aInfo = CallingMap({ - 'ip': IPAddr('192.0.2.1') - }) - self.action.ban(aInfo) - self.assertLogged('badips.com: ban', wait=True) - self.pruneLog() - # produce an error using wrong category/IP: - self.action._category = 'f2b-this-category-dont-available-test-suite-only' - aInfo['ip'] = '' - self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo) - self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e92edd48..4029c753 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -458,8 +458,6 @@ class JailReaderTest(LogCaptureTestCase): ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), ('action', '%(action_blocklist_de)s\n' - '%(action_badips_report)s\n' - '%(action_badips)s\n' 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' ), )) @@ -473,16 +471,14 @@ class JailReaderTest(LogCaptureTestCase): if len(cmd) <= 4: continue # differentiate between set and multi-set (wrop it here to single set): - if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')): + if cmd[0] == 'set' and cmd[4] == 'agent': act.append(cmd) elif cmd[0] == 'multi-set': act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent']) useragent = 'Fail2Ban/%s' % version - self.assertEqual(len(act), 4) + self.assertEqual(len(act), 2) self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) - self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) - self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) - self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) @with_tmpdir def testGlob(self, d): From 63acc862b139bb1d8b45edb0b3716044b36c7113 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2021 18:21:42 +0100 Subject: [PATCH 34/61] `action.d/nginx-block-map.conf`: reload nginx only if it is running (also avoid error in nginx-errorlog, gh-2949) and better test coverage for the action --- config/action.d/nginx-block-map.conf | 11 +++++++++-- fail2ban/tests/fail2banclienttestcase.py | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index ee702907..0de382bd 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -84,8 +84,15 @@ srv_cfg_path = /etc/nginx/ #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf srv_cmd = nginx -# first test configuration is correct, hereafter send reload signal: -blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then +# pid file (used to check nginx is running): +srv_pid = /run/nginx.pid + +# command used to check whether nginx is running and configuration is valid: +srv_is_running = [ -f "%(srv_pid)s" ] +srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt + +# first test nginx is running and configuration is correct, hereafter send reload signal: +blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi; fi; diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 03b1d7ce..d1aec5ab 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1281,7 +1281,7 @@ class Fail2banServerTest(Fail2banClientServerBase): 'backend = polling', 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', - 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + 'action = nginx-block-map[srv_cmd="echo nginx", srv_pid="%(tmp)s/f2b.pid", blck_lst_file="%(tmp)s/blck-lst.map"]', ' blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; \', email="Fail2Ban ", ' 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=]', 'filter =', @@ -1321,6 +1321,8 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check nginx reload is logged (pid of fail2ban is used to simulate success check nginx is running): + self.assertLogged("stdout: 'nginx -qt'", "stdout: 'nginx -s reload'", all=True) # check blocklist_de substitution (e. g. new-line after ): self.assertLogged( "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " From a45b1c974c969e9486326872447c33feca9ce0b2 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 2 Mar 2021 19:24:03 +0100 Subject: [PATCH 35/61] filter.d/ignorecommands/apache-fakegooglebot: added timeout parameter (default 55 seconds) - avoid fail with timeout (default 1 minute) by reverse lookup on some slow DNS services (googlebots must be resolved fast); closes gh-2951 --- .../ignorecommands/apache-fakegooglebot | 25 +++++++++++++------ fail2ban/tests/filtertestcase.py | 13 +++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index b11f0d98..8351efa2 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -6,24 +6,35 @@ # import sys from fail2ban.server.ipdns import DNSUtils, IPAddr +from threading import Thread def process_args(argv): - if len(argv) != 2: - raise ValueError("Please provide a single IP as an argument. Got: %s\n" - % (argv[1:])) + if len(argv) - 1 not in (1, 2): + raise ValueError("Usage %s ip ?timeout?. Got: %s\n" + % (argv[0], argv[1:])) ip = argv[1] if not IPAddr(ip).isValid: raise ValueError("Argument must be a single valid IP. Got: %s\n" % ip) - return ip + return argv[1:] google_ips = None -def is_googlebot(ip): +def is_googlebot(ip, timeout=55): import re - host = DNSUtils.ipToName(ip) + timeout = float(timeout or 0) + if timeout: + def ipToNameTO(host, ip, timeout): + host[0] = DNSUtils.ipToName(ip) + host = [None] + th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start() + th.join(timeout) + host = host[0] + else: + host = DNSUtils.ipToName(ip) + if not host or not re.match(r'.*\.google(bot)?\.com$', host): return False host_ips = DNSUtils.dnsToIp(host) @@ -31,7 +42,7 @@ def is_googlebot(ip): if __name__ == '__main__': # pragma: no cover try: - ret = is_googlebot(process_args(sys.argv)) + ret = is_googlebot(*process_args(sys.argv)) except ValueError as e: sys.stderr.write(str(e)) sys.exit(2) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b9b7e8aa..4f716663 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -606,13 +606,14 @@ class IgnoreIPDNS(LogCaptureTestCase): cmd = os.path.join(STOCK_CONF_DIR, "filter.d/ignorecommands/apache-fakegooglebot") ## below test direct as python module: mod = Utils.load_python_module(cmd) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "128.178.222.69"]))) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "128.178.222.69"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1", 0.1]))) bot_ips = ['66.249.66.1'] for ip in bot_ips: - self.assertTrue(mod.is_googlebot(mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd]))) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"]))) + self.assertTrue(mod.is_googlebot(*mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd]))) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd, "192.0"]))) ## via command: self.filter.ignoreCommand = cmd + " " for ip in bot_ips: @@ -624,7 +625,7 @@ class IgnoreIPDNS(LogCaptureTestCase): self.pruneLog() self.filter.ignoreCommand = cmd + " bad arguments " self.assertFalse(self.filter.inIgnoreIPList("192.0")) - self.assertLogged('Please provide a single IP as an argument.') + self.assertLogged('Usage') From 04aba6168c5f9b4b7b2bc4e2f4d128b62c3b7633 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Mar 2021 13:02:00 +0100 Subject: [PATCH 36/61] fixed typo, `--` is not expected in options declaration, so `--dump-pretty` did never work (only `--dp` is working) --- fail2ban/client/fail2bancmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 03683cad..c2f6d0be 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -192,7 +192,7 @@ class Fail2banCmdLine(): cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'conf=', 'pidfile=', 'pname=', 'socket=', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() From df5e024fb8819e90ee2b3435eb4f519e77c26495 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Mar 2021 20:13:04 +0100 Subject: [PATCH 37/61] new issue templates --- .github/ISSUE_TEMPLATE.md | 49 ---------------- .github/ISSUE_TEMPLATE/bug_report.md | 70 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 35 ++++++++++++ .github/ISSUE_TEMPLATE/filter_request.md | 59 +++++++++++++++++++ 4 files changed, 164 insertions(+), 49 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/filter_request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index cb4b4bc6..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,49 +0,0 @@ -_We will be very grateful, if your problem was described as completely as possible, -enclosing excerpts from logs (if possible within DEBUG mode, if no errors evident -within INFO mode), and configuration in particular of effected relevant settings -(e.g., with ` fail2ban-client -d | grep 'affected-jail-name' ` for a particular -jail troubleshooting). -Thank you in advance for the details, because such issues like "It does not work" -alone could not help to resolve anything! -Thanks! (remove this paragraph and other comments upon reading)_ - -### Environment: - -_Fill out and check (`[x]`) the boxes which apply. If your Fail2Ban version is outdated, -and you can't verify that the issue persists in the recent release, better seek support -from the distribution you obtained Fail2Ban from_ - -- Fail2Ban version (including any possible distribution suffixes): -- OS, including release name/version: -- [ ] Fail2Ban installed via OS/distribution mechanisms -- [ ] You have not applied any additional foreign patches to the codebase -- [ ] Some customizations were done to the configuration (provide details below is so) - -### The issue: - -_Summary here_ - -#### Steps to reproduce - -#### Expected behavior - -#### Observed behavior - -#### Any additional information - -### Configuration, dump and another helpful excerpts - -#### Any customizations done to /etc/fail2ban/ configuration -``` -``` - -#### Relevant parts of /var/log/fail2ban.log file: -_preferably obtained while running fail2ban with `loglevel = 4`_ - -``` -``` - -#### Relevant lines from monitored log files in question: - -``` -``` \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..33d94e10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,70 @@ +--- +name: Bug report +about: Report a bug within the fail2ban engines (not filters or jails) +title: '[BR]: ' +labels: bug +assignees: '' + +--- + + + +### Environment: + + + +- Fail2Ban version : +- OS, including release name/version : +- [ ] Fail2Ban installed via OS/distribution mechanisms +- [ ] You have not applied any additional foreign patches to the codebase +- [ ] Some customizations were done to the configuration (provide details below is so) + +### The issue: + + + +#### Steps to reproduce + +#### Expected behavior + +#### Observed behavior + +#### Any additional information + + +### Configuration, dump and another helpful excerpts + +#### Any customizations done to /etc/fail2ban/ configuration + +``` +``` + +#### Relevant parts of /var/log/fail2ban.log file: + + +``` +``` + +#### Relevant lines from monitored log files: + +``` +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..41812e82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,35 @@ +--- +name: Feature request +about: Suggest an idea or an enhancement for this project +title: '[RFE]: ' +labels: enhancement +assignees: '' + +--- + + + +#### Feature request type + + +#### Description + + +#### Considered alternatives + + +#### Any additional information + diff --git a/.github/ISSUE_TEMPLATE/filter_request.md b/.github/ISSUE_TEMPLATE/filter_request.md new file mode 100644 index 00000000..eeaba5ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/filter_request.md @@ -0,0 +1,59 @@ +--- +name: Filter Request +about: Request a new jail or filter to be supported or existing filter extended with new failregex. +title: '[FR]: ' +labels: filter-request +assignees: '' + +--- + + + +### Environment: + + + +- Fail2Ban version : +- OS, including release name/version : + +#### Service, project or product which log or journal should be monitored + +- Name of filter or jail in Fail2Ban (if already exists) : +- Service, project or product name, including release name/version : +- Repository or URL (if known) : +- Service type : +- Ports and protocols the service is listening : + +#### Log or journal information + + + + +- Log file name(s) : + + + +- Journal identifier or unit name : + +#### Any additional information + + +### Relevant lines from monitored log files: + +#### failures in sense of fail2ban filter (fail2ban must match): + +``` +``` + +#### legitimate messages (fail2ban should not consider as failures): + +``` +``` From 08393f9d82d4929ce3ee09bd9f3c685d38b2428d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 3 Mar 2021 20:28:27 +0100 Subject: [PATCH 38/61] Update filter_request.md --- .github/ISSUE_TEMPLATE/filter_request.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/filter_request.md b/.github/ISSUE_TEMPLATE/filter_request.md index eeaba5ae..caf02f90 100644 --- a/.github/ISSUE_TEMPLATE/filter_request.md +++ b/.github/ISSUE_TEMPLATE/filter_request.md @@ -1,6 +1,6 @@ --- -name: Filter Request -about: Request a new jail or filter to be supported or existing filter extended with new failregex. +name: Filter request +about: Request a new jail or filter to be supported or existing filter extended with new failregex title: '[FR]: ' labels: filter-request assignees: '' From f15ed356198728c18470794ce6d88fb786571dc4 Mon Sep 17 00:00:00 2001 From: Mike Gabriel Date: Thu, 25 Feb 2021 20:13:18 +0100 Subject: [PATCH 39/61] config/: Add support for filtering out detected port scans via scanlogd. --- config/filter.d/scanlogd.conf | 17 +++++++++++++++++ config/jail.conf | 3 +++ 2 files changed, 20 insertions(+) create mode 100644 config/filter.d/scanlogd.conf diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf new file mode 100644 index 00000000..65ad63f6 --- /dev/null +++ b/config/filter.d/scanlogd.conf @@ -0,0 +1,17 @@ +# Fail2Ban filter for port scans detected by scanlogd + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] + +_daemon = scanlogd + +failregex = ^%(__prefix_line)s\ to\ [\.:0-9a-f]+\ ports\ [\ \.,0-9]+,\ f.......,\ TOS\ [0-9]+,\ TTL\ [0-9]+\ \@[0-9]{1,2}:[0-9]{2}:[0-9]{2}$ + +ignoreregex = + +# Author: Mike Gabriel diff --git a/config/jail.conf b/config/jail.conf index 28f259a0..d6d8af67 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -965,3 +965,6 @@ logpath = %(apache_error_log)s # see `filter.d/traefik-auth.conf` for details and service example. port = http,https logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %{syslog_local0} From 529866b2bb41586342652af19aaca7435ab28c28 Mon Sep 17 00:00:00 2001 From: oukb <5672797+oukb@users.noreply.github.com> Date: Mon, 8 Mar 2021 19:14:28 +0300 Subject: [PATCH 40/61] nsd.conf: fix for the current log format New nsd 4.3.5 log format: | [2021-03-05 05:25:14.562] nsd[160800]: info: axfr for example.com. from 192.35.168.32 refused, no acl matches | [2021-03-06 05:24:33.223] nsd[356033]: info: axfr for localhost. from 192.35.168.160 refused, no acl matches | [2021-03-07 05:23:26.641] nsd[547893]: info: axfr for example.com. from 192.35.168.64 refused, no acl matches | [2021-03-08 05:18:54.067] nsd[739606]: info: axfr for example.com. from 192.35.168.32 refused, no acl matches --- config/filter.d/nsd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index bfd99544..9399db23 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -23,9 +23,9 @@ _daemon = nsd # Values: TEXT failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ - ^%(__prefix_line)sinfo: .* refused, no acl matches\.$ + ^%(__prefix_line)sinfo: .* refused, no acl matches ignoreregex = datepattern = {^LN-BEG}Epoch - {^LN-BEG} \ No newline at end of file + {^LN-BEG} From 725354c79315d8eac6cdc7ffa550a8acb9a01d73 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 20 Mar 2021 22:33:31 +0100 Subject: [PATCH 41/61] action info extended with new members for jail info (usable as tags in command actions): `jail.found`, `jail.found_total` - current and total found failures `jail.banned`, `jail.banned_total` - current and total bans closes #10 --- fail2ban/server/actions.py | 55 ++++++++++--------- fail2ban/tests/databasetestcase.py | 8 ++- fail2ban/tests/fail2banclienttestcase.py | 13 +++-- .../tests/files/action.d/action_checkainfo.py | 3 + 4 files changed, 48 insertions(+), 31 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 91e1ebaf..e07ffb17 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -84,7 +84,7 @@ class Actions(JailThread, Mapping): self._jail = jail self._actions = OrderedDict() ## The ban manager. - self.__banManager = BanManager() + self.banManager = BanManager() self.banEpoch = 0 self.__lastConsistencyCheckTM = 0 ## Precedence of ban (over unban), so max number of tickets banned (to call an unban check): @@ -203,7 +203,7 @@ class Actions(JailThread, Mapping): def setBanTime(self, value): value = MyTime.str2seconds(value) - self.__banManager.setBanTime(value) + self.banManager.setBanTime(value) logSys.info(" banTime: %s" % value) ## @@ -212,10 +212,10 @@ class Actions(JailThread, Mapping): # @return the time def getBanTime(self): - return self.__banManager.getBanTime() + return self.banManager.getBanTime() def getBanned(self, ids): - lst = self.__banManager.getBanList() + lst = self.banManager.getBanList() if not ids: return lst if len(ids) == 1: @@ -230,7 +230,7 @@ class Actions(JailThread, Mapping): list The list of banned IP addresses. """ - return self.__banManager.getBanList(ordered=True, withTime=withTime) + return self.banManager.getBanList(ordered=True, withTime=withTime) def addBannedIP(self, ip): """Ban an IP or list of IPs.""" @@ -282,7 +282,7 @@ class Actions(JailThread, Mapping): if db and self._jail.database is not None: self._jail.database.delBan(self._jail, ip) # Find the ticket with the IP. - ticket = self.__banManager.getTicketByID(ip) + ticket = self.banManager.getTicketByID(ip) if ticket is not None: # Unban the IP. self.__unBan(ticket) @@ -291,7 +291,7 @@ class Actions(JailThread, Mapping): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter(ipa.contains, self.__banManager.getBanList()) + ips = filter(ipa.contains, self.banManager.getBanList()) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: @@ -350,7 +350,7 @@ class Actions(JailThread, Mapping): continue # wait for ban (stop if gets inactive, pending ban or unban): bancnt = 0 - wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + wt = min(self.sleeptime, self.banManager._nextUnbanTime - MyTime.time()) logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): bancnt = self.__checkBan() @@ -397,7 +397,12 @@ class Actions(JailThread, Mapping): "ipfailures": lambda self: self._mi4ip(True).getAttempt(), "ipjailfailures": lambda self: self._mi4ip().getAttempt(), # raw ticket info: - "raw-ticket": lambda self: repr(self.__ticket) + "raw-ticket": lambda self: repr(self.__ticket), + # jail info: + "jail.banned": lambda self: self.__jail.actions.banManager.size(), + "jail.banned_total": lambda self: self.__jail.actions.banManager.getBanTotal(), + "jail.found": lambda self: self.__jail.filter.failManager.size(), + "jail.found_total": lambda self: self.__jail.filter.failManager.getFailTotal() } __slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip') @@ -494,11 +499,11 @@ class Actions(JailThread, Mapping): for ticket in tickets: bTicket = BanTicket.wrap(ticket) - btime = ticket.getBanTime(self.__banManager.getBanTime()) + btime = ticket.getBanTime(self.banManager.getBanTime()) ip = bTicket.getIP() aInfo = self._getActionInfo(bTicket) reason = {} - if self.__banManager.addBanTicket(bTicket, reason=reason): + if self.banManager.addBanTicket(bTicket, reason=reason): cnt += 1 # report ticket to observer, to check time should be increased and hereafter observer writes ban to database (asynchronous) if Observers.Main is not None and not bTicket.restored: @@ -557,7 +562,7 @@ class Actions(JailThread, Mapping): # and increase ticket time if "bantime.increment" set) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, - self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) + self.banManager.getBanTotal(), self.banManager.size(), self._jail.name) return cnt def __reBan(self, ticket, actions=None, log=True): @@ -597,7 +602,7 @@ class Actions(JailThread, Mapping): def _prolongBan(self, ticket): # prevent to prolong ticket that was removed in-between, # if it in ban list - ban time already prolonged (and it stays there): - if not self.__banManager._inBanList(ticket): return + if not self.banManager._inBanList(ticket): return # do actions : aInfo = None for name, action in self._actions.iteritems(): @@ -622,13 +627,13 @@ class Actions(JailThread, Mapping): Unban IP addresses which are outdated. """ - lst = self.__banManager.unBanList(MyTime.time(), maxCount) + lst = self.banManager.unBanList(MyTime.time(), maxCount) for ticket in lst: self.__unBan(ticket) cnt = len(lst) if cnt: logSys.debug("Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt def __flushBan(self, db=False, actions=None, stop=False): @@ -642,10 +647,10 @@ class Actions(JailThread, Mapping): log = True if actions is None: logSys.debug(" Flush ban list") - lst = self.__banManager.flushBanList() + lst = self.banManager.flushBanList() else: log = False # don't log "[jail] Unban ..." if removing actions only. - lst = iter(self.__banManager) + lst = iter(self.banManager) cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} @@ -682,7 +687,7 @@ class Actions(JailThread, Mapping): self.__unBan(ticket, actions=actions, log=log) cnt += 1 logSys.debug(" Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt def __unBan(self, ticket, actions=None, log=True): @@ -725,18 +730,18 @@ class Actions(JailThread, Mapping): logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) if flavor != "short": - banned = self.__banManager.getBanList() + banned = self.banManager.getBanList() cnt = len(banned) else: - cnt = self.__banManager.size() + cnt = self.banManager.size() ret = [("Currently banned", cnt), - ("Total banned", self.__banManager.getBanTotal())] + ("Total banned", self.banManager.getBanTotal())] if flavor != "short": ret += [("Banned IP list", banned)] if flavor == "cymru": - cymru_info = self.__banManager.getBanListExtendedCymruInfo() + cymru_info = self.banManager.getBanListExtendedCymruInfo() ret += \ - [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), - ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), - ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + [("Banned ASN list", self.banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.banManager.geBanListExtendedRIR(cymru_info))] return ret diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index a8e2ceae..298730ae 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -29,7 +29,7 @@ import tempfile import sqlite3 import shutil -from ..server.filter import FileContainer +from ..server.filter import FileContainer, Filter from ..server.mytime import MyTime from ..server.ticket import FailTicket from ..server.actions import Actions, Utils @@ -544,17 +544,21 @@ class DatabaseTest(LogCaptureTestCase): self.testAddJail() # Jail required self.jail.database = self.db self.db.addJail(self.jail) - actions = Actions(self.jail) + actions = self.jail.actions actions.add( "action_checkainfo", os.path.join(TEST_FILES_DIR, "action.d/action_checkainfo.py"), {}) + actions.banManager.setBanTotal(20) + self.jail._Jail__filter = flt = Filter(self.jail) + flt.failManager.setFailTotal(50) ticket = FailTicket("1.2.3.4") ticket.setAttempt(5) ticket.setMatches(['test', 'test']) self.jail.putFailTicket(ticket) actions._Actions__checkBan() self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) + self.assertLogged("jail info %d, %d, %d, %d" % (1, 21, 0, 50)) def testDelAndAddJail(self): self.testAddJail() # Add jail diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 244d23b0..5d322f70 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1410,8 +1410,9 @@ class Fail2banServerTest(Fail2banClientServerBase): 'jails': ( # default: '''test_action = dummy[actionstart_on_demand=1, init="start: %(__name__)s", target="%(tmp)s/test.txt", - actionban='; - echo ""; printf "=====\\n%%b\\n=====\\n\\n" "" >> ']''', + actionban='; echo "found: / , banned: / " + echo ""; printf "=====\\n%%b\\n=====\\n\\n" "" >> ', + actionstop='; echo "stats - found: , banned: "']''', # jail sendmail-auth: '[sendmail-auth]', 'backend = polling', @@ -1456,7 +1457,8 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_file(lgfn, "w+", *smaut_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( - "[sendmail-auth] Ban 192.0.2.1", "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) + "[sendmail-auth] Ban 192.0.2.1", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) _out_file(tofn) td = _read_file(tofn) # check matches (maxmatches = 2, so only 2 & 3 available): @@ -1470,7 +1472,8 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_file(lgfn, "w+", *smrej_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( - "[sendmail-reject] Ban 192.0.2.2", "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) + "[sendmail-reject] Ban 192.0.2.2", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) _out_file(tofn) td = _read_file(tofn) # check matches (no maxmatches, so all matched messages are available): @@ -1484,6 +1487,8 @@ class Fail2banServerTest(Fail2banClientServerBase): # wait a bit: self.assertLogged( "Reload finished.", + "stdout: 'stats sendmail-auth - found: 3, banned: 1'", + "stdout: 'stats sendmail-reject - found: 3, banned: 1'", "[sendmail-auth] Restore Ban 192.0.2.1", "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) # check matches again - (dbmaxmatches = 1), so it should be only last match after restart: td = _read_file(tofn) diff --git a/fail2ban/tests/files/action.d/action_checkainfo.py b/fail2ban/tests/files/action.d/action_checkainfo.py index 63dd4f5b..c5eaf0f8 100644 --- a/fail2ban/tests/files/action.d/action_checkainfo.py +++ b/fail2ban/tests/files/action.d/action_checkainfo.py @@ -8,6 +8,9 @@ class TestAction(ActionBase): self._logSys.info("ban ainfo %s, %s, %s, %s", aInfo["ipmatches"] != '', aInfo["ipjailmatches"] != '', aInfo["ipfailures"] > 0, aInfo["ipjailfailures"] > 0 ) + self._logSys.info("jail info %d, %d, %d, %d", + aInfo["jail.banned"], aInfo["jail.banned_total"], aInfo["jail.found"], aInfo["jail.found_total"] + ) def unban(self, aInfo): pass From 9bdc4be6cce7398cf51d811b7d71e1999c262fb3 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 21 Mar 2021 23:35:09 +0100 Subject: [PATCH 42/61] stability: better recognition of rotation (e. g. on hash collision, consider current size and last known position now), no hash of empty file (or not fulfilled line), etc; performance: avoid unnecessary seek to start of file and hash calculation - now it occurs only if file really rotated (ino changing or size shrinking), otherwise not earlier than in 30 seconds; avoid unneeded log-rotation in tests --- fail2ban/server/database.py | 2 +- fail2ban/server/filter.py | 109 ++++++++++++++--------- fail2ban/tests/databasetestcase.py | 7 +- fail2ban/tests/fail2banclienttestcase.py | 16 ++-- fail2ban/tests/filtertestcase.py | 19 ++-- 5 files changed, 88 insertions(+), 65 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index ed736a7a..86b0ea68 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -502,7 +502,7 @@ class Fail2BanDb(object): except TypeError: firstLineMD5 = None - if not firstLineMD5 and (pos or md5): + if firstLineMD5 is None and (pos or md5 is not None): cur.execute( "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 7ad8a462..79cde6b5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1155,6 +1155,8 @@ class FileFilter(Filter): if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Seek to find time %s (%s), file size %s", date, MyTime.time2str(date), fs) + if not fs: + return minp = container.getPos() maxp = fs tryPos = minp @@ -1281,20 +1283,25 @@ class FileContainer: self.setEncoding(encoding) self.__tail = tail self.__handler = None + self.__pos = 0 + self.__pos4hash = 0 + self.__hash = '' + self.__hashNextTime = time.time() + 30 # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') - stats = os.fstat(handler.fileno()) - self.__ino = stats.st_ino try: - firstLine = handler.readline() - # Computes the MD5 of the first line. - self.__hash = md5sum(firstLine).hexdigest() - # Start at the beginning of file if tail mode is off. - if tail: - handler.seek(0, 2) - self.__pos = handler.tell() - else: - self.__pos = 0 + stats = os.fstat(handler.fileno()) + self.__ino = stats.st_ino + if stats.st_size: + firstLine = handler.readline() + # first line available and contains new-line: + if firstLine != firstLine.rstrip('\r\n'): + # Computes the MD5 of the first line. + self.__hash = md5sum(firstLine).hexdigest() + # if tail mode scroll to the end of file + if tail: + handler.seek(0, 2) + self.__pos = handler.tell() finally: handler.close() ## shows that log is in operation mode (expecting new messages only from here): @@ -1304,6 +1311,10 @@ class FileContainer: return self.__filename def getFileSize(self): + h = self.__handler + if h is not None: + stats = os.fstat(h.fileno()) + return stats.st_size return os.path.getsize(self.__filename); def setEncoding(self, encoding): @@ -1322,38 +1333,54 @@ class FileContainer: def setPos(self, value): self.__pos = value - def open(self): - self.__handler = open(self.__filename, 'rb') - # Set the file descriptor to be FD_CLOEXEC - fd = self.__handler.fileno() - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - # Stat the file before even attempting to read it - stats = os.fstat(self.__handler.fileno()) - if not stats.st_size: - # yoh: so it is still an empty file -- nothing should be - # read from it yet - # print "D: no content -- return" - return False - firstLine = self.__handler.readline() - # Computes the MD5 of the first line. - myHash = md5sum(firstLine).hexdigest() - ## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % ( - ## self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos, - ## self.__hash != myHash or self.__ino != stats.st_ino) - ## sys.stdout.flush() - # Compare hash and inode - if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.log(logging.MSG, "Log rotation detected for %s", self.__filename) - self.__hash = myHash - self.__ino = stats.st_ino - self.__pos = 0 - # Sets the file pointer to the last position. - self.__handler.seek(self.__pos) + def open(self, forcePos=None): + h = open(self.__filename, 'rb') + try: + # Set the file descriptor to be FD_CLOEXEC + fd = h.fileno() + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + myHash = self.__hash + # Stat the file before even attempting to read it + stats = os.fstat(h.fileno()) + rotflg = stats.st_size < self.__pos or stats.st_ino != self.__ino + if rotflg or not len(myHash) or time.time() > self.__hashNextTime: + myHash = '' + firstLine = h.readline() + # Computes the MD5 of the first line (if it is complete) + if firstLine != firstLine.rstrip('\r\n'): + myHash = md5sum(firstLine).hexdigest() + self.__hashNextTime = time.time() + 30 + elif stats.st_size == self.__pos: + myHash = self.__hash + # Compare size, hash and inode + if rotflg or myHash != self.__hash: + if self.__hash != '': + logSys.log(logging.MSG, "Log rotation detected for %s, reason: %r", self.__filename, + (stats.st_size, self.__pos, stats.st_ino, self.__ino, myHash, self.__hash)) + self.__ino = stats.st_ino + self.__pos = 0 + self.__hash = myHash + # if nothing to read from file yet (empty or no new data): + if forcePos is not None: + self.__pos = forcePos + elif stats.st_size <= self.__pos: + return False + # Sets the file pointer to the last position. + h.seek(self.__pos) + # leave file open (to read content): + self.__handler = h; h = None + finally: + # close (no content or error only) + if h: + h.close(); h = None return True def seek(self, offs, endLine=True): h = self.__handler + if h is None: + self.open(offs) + h = self.__handler # seek to given position h.seek(offs, 0) # goto end of next line @@ -1394,14 +1421,12 @@ class FileContainer: self.getFileName(), self.getEncoding(), self.__handler.readline()) def close(self): - if not self.__handler is None: + if self.__handler is not None: # Saves the last position. self.__pos = self.__handler.tell() # Closes the file. self.__handler.close() self.__handler = None - ## print "D: Closed %s with pos %d" % (handler, self.__pos) - ## sys.stdout.flush() _decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 298730ae..6692b238 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -212,19 +212,20 @@ class DatabaseTest(LogCaptureTestCase): self.jail.name in self.db.getJailNames(True), "Jail not added to database") - def testAddLog(self): + def _testAddLog(self): self.testAddJail() # Jail required _, filename = tempfile.mkstemp(".log", "Fail2BanDb_") self.fileContainer = FileContainer(filename, "utf-8") - self.db.addLog(self.jail, self.fileContainer) + pos = self.db.addLog(self.jail, self.fileContainer) + self.assertTrue(pos is None); # unknown previously self.assertIn(filename, self.db.getLogPaths(self.jail)) os.remove(filename) def testUpdateLog(self): - self.testAddLog() # Add log file + self._testAddLog() # Add log file # Write some text filename = self.fileContainer.getFileName() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 5d322f70..0cbda94f 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -230,7 +230,7 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n)) if create_before_start: for n in create_before_start: - _write_file(n % {'tmp': tmp}, 'w', '') + _write_file(n % {'tmp': tmp}, 'w') # parameters (sock/pid and config, increase verbosity, set log, etc.): vvv, llev = (), "INFO" if unittest.F2B.log_level < logging.INFO: # pragma: no cover @@ -937,10 +937,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'broken-jail' skipped, because of wrong configuration", all=True) # enable both jails, 3 logs for jail1, etc... - # truncate test-log - we should not find unban/ban again by reload: self.pruneLog("[test-phase 1b]") _write_jail_cfg(actions=[1,2]) - _write_file(test1log, "w+") if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(test1log) self.execCmd(SUCCESS, startparams, "reload") @@ -1003,7 +1001,7 @@ class Fail2banServerTest(Fail2banClientServerBase): self.pruneLog("[test-phase 2b]") # write new failures: - _write_file(test2log, "w+", *( + _write_file(test2log, "a+", *( (str(int(MyTime.time())) + " error 403 from 192.0.2.2: test 2",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.3: test 2",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 + @@ -1062,10 +1060,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertEqual(self.execCmdDirect(startparams, 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) - # rotate logs: - _write_file(test1log, "w+") - _write_file(test2log, "w+") - # restart jail without unban all: self.pruneLog("[test-phase 2c]") self.execCmd(SUCCESS, startparams, @@ -1183,7 +1177,7 @@ class Fail2banServerTest(Fail2banClientServerBase): # now write failures again and check already banned (jail1 was alive the whole time) and new bans occurred (jail1 was alive the whole time): self.pruneLog("[test-phase 5]") - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 5",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.5: test 5",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.6: test 5",) * 3 @@ -1469,7 +1463,7 @@ class Fail2banServerTest(Fail2banClientServerBase): self.pruneLog("[test-phase sendmail-reject]") # write log: - _write_file(lgfn, "w+", *smrej_msg) + _write_file(lgfn, "a+", *smrej_msg) # wait and check it caused banned (and dump in the test-file): self.assertLogged( "[sendmail-reject] Ban 192.0.2.2", "stdout: 'found: 0 / 3, banned: 1 / 1'", @@ -1597,7 +1591,7 @@ class Fail2banServerTest(Fail2banClientServerBase): wakeObs = False _observer_wait_before_incrban(lambda: wakeObs) # write again (IP already bad): - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.11: I'm very bad \"hacker\" `` $(echo test)",) * 2 )) # wait for ban: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 28b2d357..799adfd3 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -195,7 +195,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, f, o) -def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line="", lines=None): """Copy lines from one file to another (which might be already open) Returns open fout @@ -212,9 +212,9 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line fin.readline() # Read i = 0 - lines = [] + if not lines: lines = [] while n is None or i < n: - l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break lines.append(l) @@ -222,6 +222,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) + DefLogSys.debug(' ++ write %d test lines', len(lines)) fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases @@ -253,7 +254,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Read/Write i = 0 while n is None or i < n: - l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break journal.send(MESSAGE=l.strip(), **fields) @@ -1136,13 +1137,15 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1, + lines=["Aug 14 11:59:59 [logrotate] rotation 1"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3, + lines=["Aug 14 11:59:59 [logrotate] rotation 2"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1196,7 +1199,7 @@ def get_monitor_failures_testcase(Filter_): os.rename(tmpsub1, tmpsub2 + 'a') os.mkdir(tmpsub1) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 1"]) self.file.close() self._wait4failures(2) @@ -1207,7 +1210,7 @@ def get_monitor_failures_testcase(Filter_): os.mkdir(tmpsub1) self.waitForTicks(2) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 2"]) self.file.close() self._wait4failures(3) From 343ccd7e8a139a4c6526682290dcbf14b4384973 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 21 Mar 2021 23:35:38 +0100 Subject: [PATCH 43/61] small optimization --- fail2ban/server/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 294d147f..8483b013 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -332,11 +332,9 @@ class Utils(): timeout_expr = lambda: time.time() > time0 else: timeout_expr = timeout - if not interval: - interval = Utils.DEFAULT_SLEEP_INTERVAL if timeout_expr(): break - stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME) + stm = min(stm + (interval or Utils.DEFAULT_SLEEP_INTERVAL), Utils.DEFAULT_SLEEP_TIME) time.sleep(stm) return ret From e587526ede759977b8b483deeaa644f45f97cea3 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Mar 2021 00:55:28 +0100 Subject: [PATCH 44/61] tests: add missing constraint (causing incomplete comparison in below cycle if fewer lines as expected was found) --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 4f716663..3cc17fb1 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -188,6 +188,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): utest.assertEqual(len(found), 1) _assert_equal_entries(utest, found[0], output, count) else: + utest.assertEqual(len(found), len(output)) # sort by string representation of ip (multiple failures with different ips): found = sorted(found, key=lambda x: str(x)) output = sorted(output, key=lambda x: str(x)) From 996920cdaa7b678afc64d1bef6a07ed857c81468 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Mar 2021 00:49:55 +0100 Subject: [PATCH 45/61] in operation mode the filter reads only complete lines (ended with new-line) now, otherwise it would wait for end of line (for its completion) --- fail2ban/server/filter.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 79cde6b5..845b069d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1131,14 +1131,14 @@ class FileFilter(Filter): while not self.idle: line = log.readline() if not self.active: break; # jail has been stopped - if not line: + if line is None: # The jail reached the bottom, simply set in operation for this log # (since we are first time at end of file, growing is only possible after modifications): log.inOperation = True break # acquire in operation from log and process: self.inOperation = inOperation if inOperation is not None else log.inOperation - self.processLineAndAdd(line.rstrip('\r\n')) + self.processLineAndAdd(line) finally: log.close() db = self.jail.database @@ -1180,8 +1180,8 @@ class FileFilter(Filter): dateTimeMatch = None nextp = None while True: - line = container.readline() - if not line: + line = container.readline(False) + if line is None: break (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: @@ -1295,7 +1295,7 @@ class FileContainer: if stats.st_size: firstLine = handler.readline() # first line available and contains new-line: - if firstLine != firstLine.rstrip('\r\n'): + if firstLine != firstLine.rstrip(b'\r\n'): # Computes the MD5 of the first line. self.__hash = md5sum(firstLine).hexdigest() # if tail mode scroll to the end of file @@ -1348,7 +1348,7 @@ class FileContainer: myHash = '' firstLine = h.readline() # Computes the MD5 of the first line (if it is complete) - if firstLine != firstLine.rstrip('\r\n'): + if firstLine != firstLine.rstrip(b'\r\n'): myHash = md5sum(firstLine).hexdigest() self.__hashNextTime = time.time() + 30 elif stats.st_size == self.__pos: @@ -1414,15 +1414,35 @@ class FileContainer: line = line.decode(enc, 'replace') return line - def readline(self): + def readline(self, complete=True): + """Read line from file + + In opposite to pythons readline it doesn't return new-line, + so returns either the line if line is complete (and complete=True) or None + if line is not complete (and complete=True) or there is no content to read. + If line is complete (and complete is True), it also shift current known + position to begin of next line. + """ if self.__handler is None: return "" + rl = self.__handler.readline() + if rl == b'': + return None + # trim new-line here and check the line was written complete (contains a new-line): + l = rl.rstrip(b'\r\n') + if self.inOperation and complete: + if l == rl: + # not fulfilled - seek back and return: + self.__handler.seek(self.__pos, 0) + return None + # shift position (to be able to seek back above): + self.__pos += len(rl) return FileContainer.decode_line( - self.getFileName(), self.getEncoding(), self.__handler.readline()) + self.getFileName(), self.getEncoding(), l) def close(self): if self.__handler is not None: - # Saves the last position. + # Saves the last real position. self.__pos = self.__handler.tell() # Closes the file. self.__handler.close() From d135aeea16343152e47191559b17912df9373705 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 14:12:11 +0100 Subject: [PATCH 46/61] fixes restore of original logging withing tests (`LogCaptureTestCase.tearDown`) - python 3 seemed still to log wordy after tear down (setting of log.level does not restore the level for related log objects - e. g. for logger of `fail2ban.jail` etc, so `fail2ban-testcases '(testVersion|testLongName).*servertest'` generating messages in stdout handler in testLongName) --- fail2ban/tests/servertestcase.py | 10 +++++----- fail2ban/tests/utils.py | 8 +++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index b7b9d802..fc505552 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -66,9 +66,12 @@ class TestServer(Server): class TransmitterBase(LogCaptureTestCase): + TEST_SRV_CLASS = TestServer + def setUp(self): """Call before every test case.""" super(TransmitterBase, self).setUp() + self.server = self.TEST_SRV_CLASS() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -157,10 +160,6 @@ class TransmitterBase(LogCaptureTestCase): class Transmitter(TransmitterBase): - def setUp(self): - self.server = TestServer() - super(Transmitter, self).setUp() - def testServerIsNotStarted(self): # so far isStarted only tested but not used otherwise # and here we don't really .start server @@ -893,8 +892,9 @@ class Transmitter(TransmitterBase): class TransmitterLogging(TransmitterBase): + TEST_SRV_CLASS = Server + def setUp(self): - self.server = Server() super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 921427db..e2c4bccc 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -47,7 +47,7 @@ from ..server import asyncserver from ..version import version -logSys = getLogger(__name__) +logSys = getLogger("fail2ban") TEST_NOW = 1124013600 @@ -126,9 +126,6 @@ def getOptParser(doc=""): def initProcess(opts): # Logger: - global logSys - logSys = getLogger("fail2ban") - llev = None if opts.log_level is not None: # pragma: no cover # so we had explicit settings @@ -777,8 +774,9 @@ class LogCaptureTestCase(unittest.TestCase): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() self.pruneLog() + self._log.close() logSys.handlers = self._old_handlers - logSys.level = self._old_level + logSys.setLevel(self._old_level) super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): From cbac7c176a93207027a53aa4591342f7268a1547 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Mar 2021 12:57:42 +0100 Subject: [PATCH 47/61] readline fixed to consider interim new-line character as part of code point in multi-byte logs (e. g. unicode: utf-16be, utf-16le); suppress warning "Error decoding line" for incomplete line (produced by not fully read multi-byte new-line character at end of data); added test coverage for such logs --- fail2ban/server/filter.py | 59 ++++++++++++++++++++++------- fail2ban/tests/filtertestcase.py | 64 +++++++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 23 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 845b069d..16279627 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1398,6 +1398,9 @@ class FileContainer: try: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: + # avoid warning if got incomplete end of line (e. g. '\n' in "...[0A" followed by "00]..." for utf-16le: + if (e.end == len(line) and line[e.start] in b'\r\n'): + return line[0:e.start].decode(enc, 'replace') global _decode_line_warn lev = 7 if not _decode_line_warn.get(filename, 0): @@ -1406,9 +1409,9 @@ class FileContainer: logSys.log(lev, "Error decoding line from '%s' with '%s'.", filename, enc) if logSys.getEffectiveLevel() <= lev: - logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", + logSys.log(lev, + "Consider setting logencoding to appropriate encoding for this jail. " + "Continuing to process line ignoring invalid characters: %r", line) # decode with replacing error chars: line = line.decode(enc, 'replace') @@ -1422,23 +1425,51 @@ class FileContainer: if line is not complete (and complete=True) or there is no content to read. If line is complete (and complete is True), it also shift current known position to begin of next line. + + Also it is safe against interim new-line bytes (e. g. part of multi-byte char) + in given encoding. """ if self.__handler is None: return "" - rl = self.__handler.readline() - if rl == b'': + # read raw bytes up to \n char: + b = self.__handler.readline() + if not b: return None - # trim new-line here and check the line was written complete (contains a new-line): - l = rl.rstrip(b'\r\n') - if self.inOperation and complete: - if l == rl: + bl = len(b) + # convert to log-encoding (new-line char could disappear if it is part of multi-byte sequence): + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if complete: + if l == r: + # try to fill buffer in order to find line-end in log encoding: + fnd = 0 + while 1: + r = self.__handler.readline() + if not r: + break + b += r + bl += len(r) + # convert to log-encoding: + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # ensure new-line is not in the middle (buffered 2 strings, e. g. in utf-16le it is "...[0A"+"00]..."): + e = r.find('\n') + if e >= 0 and e != len(r)-1: + l, r = r[0:e], r[0:e+1] + # back to bytes and get offset to seek after NL: + r = r.encode(self.getEncoding(), 'replace') + self.__handler.seek(-bl+len(r), 1) + return l + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if l != r: + return l # not fulfilled - seek back and return: - self.__handler.seek(self.__pos, 0) + self.__handler.seek(-bl, 1) return None - # shift position (to be able to seek back above): - self.__pos += len(rl) - return FileContainer.decode_line( - self.getFileName(), self.getEncoding(), l) + return l def close(self): if self.__handler is not None: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 799adfd3..f8621f29 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -644,6 +644,19 @@ class LogFile(LogCaptureTestCase): self.filter = FilterPoll(None) self.assertRaises(IOError, self.filter.addLogPath, LogFile.MISSING) + def testDecodeLineWarn(self): + # incomplete line (missing byte at end), warning is suppressed: + l = u"correct line\n" + r = l.encode('utf-16le') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) + self.assertNotLogged('Error decoding line') + # complete line (incorrect surrogate in the middle), warning is there: + r = b"incorrect \xc8\x0a line\n" + l = r.decode('utf-8', 'replace') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-8', r), l) + self.assertLogged('Error decoding line') + class LogFileFilterPoll(unittest.TestCase): @@ -1633,16 +1646,49 @@ class GetFailures(LogCaptureTestCase): def testCRLFFailures01(self): # We first adjust logfile/failures to end with CR+LF fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') - # poor man unix2dos: - fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') - for l in fin.read().splitlines(): - fout.write(l + b'\r\n') - fin.close() - fout.close() + try: + # poor man unix2dos: + fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') + for l in fin.read().splitlines(): + fout.write(l + b'\r\n') + fin.close() + fout.close() - # now see if we should be getting the "same" failures - self.testGetFailures01(filename=fname) - _killfile(fout, fname) + # now see if we should be getting the "same" failures + self.testGetFailures01(filename=fname) + finally: + _killfile(fout, fname) + + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + tm = int(time.time()) + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + ): + fout.write(l.encode(enc)) + fout.close() + + self.filter.setLogEncoding(enc) + self.filter.addLogPath(fname, autoSeek=0) + self.filter.setDatePattern((r'^EPOCH',)) + self.filter.addFailRegex(r"Failed .* from ") + self.filter.getFailures(fname) + self.assertLogged( + "[DummyJail] Found 192.0.2.1", + "[DummyJail] Found 192.0.2.2", all=True, wait=True) + finally: + _killfile(fout, fname) + self.filter.delLogPath(fname) + # must find 4 failures and generate 2 tickets (2 IPs with each 2 failures): + self.assertEqual(self.filter.failManager.getFailCount(), (2, 4)) def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, From 9659033523e27f62b2c6ad7db5e1671e58d4f045 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 16:19:06 +0100 Subject: [PATCH 48/61] fail2ban-regex: reimplemented log-file iterator - uses FileContainer facilities now instead of direct read from file and decode; fail2banregextestcase.py extended to cover proper line-ending handling by interim NL char as part of multi-byte encodings (utf-16be, utf-16le) --- fail2ban/client/fail2banregex.py | 13 ++++---- fail2ban/server/filter.py | 5 ++- fail2ban/tests/fail2banregextestcase.py | 44 +++++++++++++++++++++---- fail2ban/tests/filtertestcase.py | 2 +- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 90e178f9..26e7394a 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -289,9 +289,6 @@ class Fail2banRegex(object): def output(self, line): if not self._opts.out: output(line) - def decode_line(self, line): - return FileContainer.decode_line('', self._encoding, line) - def encode_line(self, line): return line.encode(self._encoding, 'ignore') @@ -724,8 +721,12 @@ class Fail2banRegex(object): return True def file_lines_gen(self, hdlr): - for line in hdlr: - yield self.decode_line(line) + while 1: + line = hdlr.readline() + if line is None: + break + yield line + hdlr.close() def start(self, args): @@ -745,7 +746,7 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = open(cmd_log, 'rb') + hdlr = FileContainer(cmd_log, self._encoding, doOpen=True) self.output( "Use log file : %s" % cmd_log ) self.output( "Use encoding : %s" % self._encoding ) test_lines = self.file_lines_gen(hdlr) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 16279627..9a2df255 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1278,7 +1278,7 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail=False): + def __init__(self, filename, encoding, tail=False, doOpen=False): self.__filename = filename self.setEncoding(encoding) self.__tail = tail @@ -1289,6 +1289,9 @@ class FileContainer: self.__hashNextTime = time.time() + 30 # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') + if doOpen: # fail2ban-regex only (don't need to reopen it and check for rotation) + self.__handler = handler + return try: stats = os.fstat(handler.fileno()) self.__ino = stats.st_ino diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 0a33fd9d..32acc0ae 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -25,6 +25,7 @@ __license__ = "GPL" import os import sys +import tempfile import unittest from ..client import fail2banregex @@ -80,6 +81,11 @@ def _test_exec_command_line(*args): sys.stderr = _org['stderr'] return _exit_code +def _reset(): + # reset global warn-counter: + from ..server.filter import _decode_line_warn + _decode_line_warn.clear() + STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" @@ -122,6 +128,7 @@ class Fail2banRegexTest(LogCaptureTestCase): """Call before every test case.""" LogCaptureTestCase.setUp(self) setUpMyTime() + _reset() def tearDown(self): """Call after every test case.""" @@ -454,14 +461,8 @@ class Fail2banRegexTest(LogCaptureTestCase): FILENAME_ZZZ_GEN, FILENAME_ZZZ_GEN )) - def _reset(self): - # reset global warn-counter: - from ..server.filter import _decode_line_warn - _decode_line_warn.clear() - def testWronChar(self): unittest.F2B.SkipIfCfgMissing(stock=True) - self._reset() self.assertTrue(_test_exec( "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", @@ -477,7 +478,6 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWronCharDebuggex(self): unittest.F2B.SkipIfCfgMissing(stock=True) - self._reset() self.assertTrue(_test_exec( "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", @@ -490,6 +490,36 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + ): + fout.write(l.encode(enc)) + fout.close() + + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "--encoding", enc, + "--datepattern", r"^EPOCH", + fname, r"Failed .* from ", + )) + + self.assertLogged(" encoding : %s" % enc, + "Lines: 2 lines, 0 ignored, 2 matched, 0 missed", all=True) + self.assertNotLogged("Missed line(s)") + finally: + fout.close() + os.unlink(fname) + def testExecCmdLine_Usage(self): self.assertNotEqual(_test_exec_command_line(), 0) self.pruneLog() diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index f8621f29..319f16de 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1660,7 +1660,7 @@ class GetFailures(LogCaptureTestCase): _killfile(fout, fname) def testNLCharAsPartOfUniChar(self): - fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): for enc in ('utf-16be', 'utf-16le'): self.pruneLog("[test-phase encoding=%s]" % enc) From ccf4f3a07dbe5e21dbca1e0f3e702cbdaaeabaf3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 17:22:05 +0100 Subject: [PATCH 49/61] amend with common log-file iterator in fail2ban-regex and test-suite (in sample regex factory also) --- fail2ban/client/fail2banregex.py | 12 ++---------- fail2ban/server/filter.py | 17 ++++++++++++++--- fail2ban/tests/samplestestcase.py | 15 ++++++++------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 26e7394a..5921dfdd 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -720,14 +720,6 @@ class Fail2banRegex(object): return True - def file_lines_gen(self, hdlr): - while 1: - line = hdlr.readline() - if line is None: - break - yield line - hdlr.close() - def start(self, args): cmd_log, cmd_regex = args[:2] @@ -746,10 +738,10 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = FileContainer(cmd_log, self._encoding, doOpen=True) + test_lines = FileContainer(cmd_log, self._encoding, doOpen=True) + self.output( "Use log file : %s" % cmd_log ) self.output( "Use encoding : %s" % self._encoding ) - test_lines = self.file_lines_gen(hdlr) except IOError as e: # pragma: no cover output( e ) return False diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9a2df255..e16d86c9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -1280,6 +1280,7 @@ class FileContainer: def __init__(self, filename, encoding, tail=False, doOpen=False): self.__filename = filename + self.waitForLineEnd = True self.setEncoding(encoding) self.__tail = tail self.__handler = None @@ -1469,9 +1470,10 @@ class FileContainer: l = r.rstrip('\r\n') if l != r: return l - # not fulfilled - seek back and return: - self.__handler.seek(-bl, 1) - return None + if self.waitForLineEnd: + # not fulfilled - seek back and return: + self.__handler.seek(-bl, 1) + return None return l def close(self): @@ -1482,6 +1484,15 @@ class FileContainer: self.__handler.close() self.__handler = None + def __iter__(self): + return self + def next(self): + line = self.readline() + if line is None: + self.close() + raise StopIteration + return line + _decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 5a72ffa9..b33b46c1 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -23,7 +23,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import datetime -import fileinput import inspect import json import os @@ -156,12 +155,15 @@ def testSampleRegexsFactory(name, basedir): i = 0 while i < len(filenames): filename = filenames[i]; i += 1; - logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", - filename), mode='rb') + logFile = FileContainer(os.path.join(TEST_FILES_DIR, "logs", + filename), 'UTF-8', doOpen=True) + # avoid errors if no NL char at end of test log-file: + logFile.waitForLineEnd = False ignoreBlock = False + lnnum = 0 for line in logFile: - line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) + lnnum += 1 jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: try: @@ -201,9 +203,8 @@ def testSampleRegexsFactory(name, basedir): # failJSON - faildata contains info of the failure to check it. except ValueError as e: # pragma: no cover - we've valid json's raise ValueError("%s: %s:%i" % - (e, logFile.filename(), logFile.filelineno())) + (e, logFile.getFileName(), lnnum)) line = next(logFile) - line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable @@ -298,7 +299,7 @@ def testSampleRegexsFactory(name, basedir): import pprint raise AssertionError("%s: %s on: %s:%i, line:\n %s\nregex (%s):\n %s\n" "faildata: %s\nfail: %s" % ( - fltName, e, logFile.filename(), logFile.filelineno(), + fltName, e, logFile.getFileName(), lnnum, line, failregex, regexList[failregex] if failregex != -1 else None, '\n'.join(pprint.pformat(faildata).splitlines()), '\n'.join(pprint.pformat(fail).splitlines()))) From 4b17dddc2389a71bba15be9a3129b03665f853b0 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 25 Mar 2021 12:05:09 +0100 Subject: [PATCH 50/61] update ChangeLog --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index e5e7b485..4fe8acdc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,12 +18,17 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition different from 0) in case of unsane environment. ### Fixes +* readline fixed to consider interim new-line character as part of code point in multi-byte logs + (e. g. unicode encoding like utf-16be, utf-16le); * `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) ### New Features and Enhancements * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) +* better recognition of log rotation, better performance by reopen: avoid unnecessary seek to begin of file + (and hash calculation) +* file filter reads only complete lines (ended with new-line) now, so waits for end of line (for its completion) ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools From b259e819114e587e0e9b17253860dd48f9579997 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Mar 2021 17:23:14 +0100 Subject: [PATCH 51/61] test-suite: skip testFQDN if no network --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 319f16de..27f5695e 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -2252,6 +2252,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2)) def testFQDN(self): + unittest.F2B.SkipIfNoNetwork() sname = DNSUtils.getHostname(fqdn=False) lname = DNSUtils.getHostname(fqdn=True) # FQDN is not localhost if short hostname is not localhost too (or vice versa): From 5aa20c30d8b283970e4b3ee0938f729850ebb618 Mon Sep 17 00:00:00 2001 From: Markus Felten Date: Fri, 12 Feb 2021 09:29:03 +0100 Subject: [PATCH 52/61] fix: add journalmatch to nginx filters --- config/filter.d/nginx-bad-request.conf | 2 ++ config/filter.d/nginx-botsearch.conf | 4 +++- config/filter.d/nginx-http-auth.conf | 2 ++ config/filter.d/nginx-limit-req.conf | 3 +++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index 2b8f5ab6..12c14ab7 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -11,4 +11,6 @@ datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,] ^[^\[]*\[({DATE}) {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # Author: Jan Przybylak diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 0be895b2..2bd23072 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -17,7 +17,9 @@ datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,] ^[^\[]*\[({DATE}) {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV Notes: # Based on apache-botsearch filter # -# Author: Frantisek Sumsal \ No newline at end of file +# Author: Frantisek Sumsal diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 93341cd2..3a2e314b 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -10,6 +10,8 @@ ignoreregex = datepattern = {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV NOTES: # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Extensive search of all nginx auth failures not done yet. diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index e23548ab..2f45e831 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -44,3 +44,6 @@ failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by ignoreregex = datepattern = {^LN-BEG} + +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + From 9eaa2322b0913c805cfa7d45be96d0067ffe61b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Olschewsky?= Date: Mon, 24 Feb 2020 16:35:42 +0100 Subject: [PATCH 53/61] Filter and Defaults for Microsoft SQL Server --- config/filter.d/mssql-auth.conf | 15 +++++++++++++++ config/jail.conf | 8 ++++++++ fail2ban/tests/files/logs/mssql-auth | 3 +++ 3 files changed, 26 insertions(+) create mode 100644 config/filter.d/mssql-auth.conf create mode 100644 fail2ban/tests/files/logs/mssql-auth diff --git a/config/filter.d/mssql-auth.conf b/config/filter.d/mssql-auth.conf new file mode 100644 index 00000000..a1813c83 --- /dev/null +++ b/config/filter.d/mssql-auth.conf @@ -0,0 +1,15 @@ +# Fail2Ban filter for failed MSSQL Server authentication attempts + +[Definition] + +failregex = Logon\s+Login failed for user ('.*')(.*)\[CLIENT: \]$ + + +# DEV Notes: +# Tested with SQL Server 2019 on Ubuntu 18.04 +# +# Example: +# 2020-02-24 14:48:55.12 Logon Login failed for user 'root'. Reason: Could not find a login matching the name provided. [CLIENT: 127.0.0.1] +# +# Author: Rüdiger Olschewsky +# \ No newline at end of file diff --git a/config/jail.conf b/config/jail.conf index 20958d11..aeff3616 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -786,6 +786,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation diff --git a/fail2ban/tests/files/logs/mssql-auth b/fail2ban/tests/files/logs/mssql-auth new file mode 100644 index 00000000..5a97f53e --- /dev/null +++ b/fail2ban/tests/files/logs/mssql-auth @@ -0,0 +1,3 @@ +2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 212.96.131.253] +2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 148.86.203.199] +2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 105.254.136.171] \ No newline at end of file From 7f38b80d351cc51cd3273ad93cd1939a4b1abe1e Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 3 Apr 2021 20:16:47 +0200 Subject: [PATCH 54/61] precise regex (left anchor and fewer catch-all's); fixed tests (added failJSON and more tests for some corner-cases around new RE) --- config/filter.d/mssql-auth.conf | 2 +- fail2ban/tests/files/logs/mssql-auth | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/config/filter.d/mssql-auth.conf b/config/filter.d/mssql-auth.conf index a1813c83..65bbd917 100644 --- a/config/filter.d/mssql-auth.conf +++ b/config/filter.d/mssql-auth.conf @@ -2,7 +2,7 @@ [Definition] -failregex = Logon\s+Login failed for user ('.*')(.*)\[CLIENT: \]$ +failregex = ^\s*Logon\s+Login failed for user '(?:[^']*|.*)'\. [^'\[]+\[CLIENT: \]$ # DEV Notes: diff --git a/fail2ban/tests/files/logs/mssql-auth b/fail2ban/tests/files/logs/mssql-auth index 5a97f53e..1c9b65ec 100644 --- a/fail2ban/tests/files/logs/mssql-auth +++ b/fail2ban/tests/files/logs/mssql-auth @@ -1,3 +1,11 @@ -2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 212.96.131.253] -2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 148.86.203.199] -2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 105.254.136.171] \ No newline at end of file +# failJSON: { "time": "2020-02-24T16:05:21", "match": true , "host": "192.0.2.1" } +2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.1] +# failJSON: { "time": "2020-02-24T16:30:25", "match": true , "host": "192.0.2.2" } +2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.2] +# failJSON: { "time": "2020-02-24T16:31:12", "match": true , "host": "192.0.2.3" } +2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 192.0.2.3] + +# failJSON: { "time": "2020-02-24T16:31:26", "match": true , "host": "192.0.2.4", "user":"O'Leary" } +2020-02-24 16:31:26.01 Logon Login failed for user 'O'Leary'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.4] +# failJSON: { "time": "2020-02-24T16:31:26", "match": false, "desc": "test injection in possibly unescaped foreign input" } +2020-02-24 16:31:26.02 Wrong data received: Logon Login failed for user 'test'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.5] From 1215cb28ac6a79acda2699694556bc1a6cef1e45 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 20:58:26 +0200 Subject: [PATCH 55/61] Update nsd --- fail2ban/tests/files/logs/nsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/nsd b/fail2ban/tests/files/logs/nsd index a33a52a9..63c162e9 100644 --- a/fail2ban/tests/files/logs/nsd +++ b/fail2ban/tests/files/logs/nsd @@ -2,3 +2,5 @@ [1387288694] nsd[7745]: info: ratelimit block example.com. type any target 192.0.2.0/24 query 192.0.2.105 TYPE255 # failJSON: { "time": "2013-12-18T07:42:15", "match": true , "host": "192.0.2.115" } [1387348935] nsd[23600]: info: axfr for zone domain.nl. from client 192.0.2.115 refused, no acl matches. +# failJSON: { "time": "2021-03-05T05:25:14", "match": true , "host": "192.0.2.32", "desc": "new format, no client after from, no dot at end, gh-2965" } +[2021-03-05 05:25:14.562] nsd[160800]: info: axfr for example.com. from 192.0.2.32 refused, no acl matches From a838deba7f8066d50984ed24983fb90f15edfaaf Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 21:00:14 +0200 Subject: [PATCH 56/61] restore anchor (e. g. catch all in the middle), dot is optional now, RE rewritten a bit more precise --- config/filter.d/nsd.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index 9399db23..0589c16c 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -22,8 +22,8 @@ _daemon = nsd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ - ^%(__prefix_line)sinfo: .* refused, no acl matches +failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ + ^%(__prefix_line)sinfo: .* from(?: client)? refused, no acl matches\.?$ ignoreregex = From 0c4d356d118d1282d8bdbed21cb514f02953e65f Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:10:51 +0200 Subject: [PATCH 57/61] added test log-file --- fail2ban/tests/files/logs/scanlogd | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 fail2ban/tests/files/logs/scanlogd diff --git a/fail2ban/tests/files/logs/scanlogd b/fail2ban/tests/files/logs/scanlogd new file mode 100644 index 00000000..5a97c578 --- /dev/null +++ b/fail2ban/tests/files/logs/scanlogd @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-03-05T21:44:43", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:43 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 80, 81, 83, 88, 99, 443, 1080, 3128, ..., f????uxy, TOS 00, TTL 49 @20:44:43 +# failJSON: { "time": "2005-03-05T21:44:44", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:44 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 497, 515, 544, 543, 464, 513, ..., fSrpauxy, TOS 00 @09:04:25 +# failJSON: { "time": "2005-03-05T21:44:45", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:45 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 593, 548, 636, 646, 625, 631, ..., fSrpauxy, TOS 00, TTL 239 @17:34:00 +# failJSON: { "time": "2005-03-05T21:44:46", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:46 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 22, 26, 37, 80, 25, 79, ..., fSrpauxy, TOS 00 @22:38:37 From 14edeed310ea5d74a13ec72d415b934cb4f7beb1 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:24:55 +0200 Subject: [PATCH 58/61] fixed regex (don't need to match whole line, e. g. every port etc) --- config/filter.d/scanlogd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf index 65ad63f6..15fea329 100644 --- a/config/filter.d/scanlogd.conf +++ b/config/filter.d/scanlogd.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = scanlogd -failregex = ^%(__prefix_line)s\ to\ [\.:0-9a-f]+\ ports\ [\ \.,0-9]+,\ f.......,\ TOS\ [0-9]+,\ TTL\ [0-9]+\ \@[0-9]{1,2}:[0-9]{2}:[0-9]{2}$ +failregex = ^%(__prefix_line)s(?::) to \S+ ports\b ignoreregex = From 977dfe4bd762474e4e2d077e258c49772a48f6f3 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:29:16 +0200 Subject: [PATCH 59/61] small amend: sport after saddr is optional format of message: saddr[:sport] to daddr [and others,] ports port[, port...], ..., flags[, TOS TOS][, TTL TTL] @HH:MM:SS --- config/filter.d/scanlogd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf index 15fea329..d3fe78b0 100644 --- a/config/filter.d/scanlogd.conf +++ b/config/filter.d/scanlogd.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = scanlogd -failregex = ^%(__prefix_line)s(?::) to \S+ ports\b +failregex = ^%(__prefix_line)s(?::)? to \S+ ports\b ignoreregex = From 2d51240b3e9e19a4259c03058ddde6ce8698397b Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sat, 3 Apr 2021 23:33:49 +0200 Subject: [PATCH 60/61] correction for default log interpolation and added allports banaction --- config/jail.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index d6d8af67..52cc161d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -967,4 +967,5 @@ port = http,https logpath = /var/log/traefik/access.log [scanlogd] -logpath = %{syslog_local0} +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s From 7579072e3b1d0663ac50ff067a953d4fe8c28ac7 Mon Sep 17 00:00:00 2001 From: Michele Mondelli Date: Thu, 31 Dec 2020 17:03:19 +0100 Subject: [PATCH 61/61] docs: fix typos --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index b4b9f249..cd802a57 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -67,7 +67,7 @@ before = paths-debian.conf # more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : #bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) -# "bantime.multipliers" used to calculate next value of ban time instead of formula, coresponding +# "bantime.multipliers" used to calculate next value of ban time instead of formula, corresponding # previously ban count and given "bantime.factor" (for multipliers default is 1); # following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, # always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours @@ -77,7 +77,7 @@ before = paths-debian.conf #bantime.multipliers = 1 5 30 60 300 720 1440 2880 # "bantime.overalljails" (if true) specifies the search of IP in the database will be executed -# cross over all jails, if false (dafault), only current jail of the ban IP will be searched +# cross over all jails, if false (default), only current jail of the ban IP will be searched #bantime.overalljails = false # --------------------