From bb0f732ae69894b22306dd7efa213513e3acd8a2 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Jan 2020 20:38:26 +0100 Subject: [PATCH 01/15] version bump (master is 1.0.x-dev now) --- .github/PULL_REQUEST_TEMPLATE.md | 3 ++- ChangeLog | 10 ++++++++++ fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-python.1 | 2 +- man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 8 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3a17ccc2..350d6ee2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,8 @@ Before submitting your PR, please review the following checklist: - [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement - against 0.9.x series, choose `master` branch + against certain release version, choose `0.9`, `0.10` or `0.11` branch, + for dev-edition use `master` branch - [ ] **CONSIDER adding a unit test** if your PR resolves an issue - [ ] **LIST ISSUES** this PR resolves - [ ] **MAKE SURE** this PR doesn't break existing tests diff --git a/ChangeLog b/ChangeLog index cc0c6608..3f9c11b1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,16 @@ Fail2Ban: Changelog =================== +ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition +----------- + +### Fixes + +### New Features + +### Enhancements + + ver. 0.11.1 (2020/01/11) - this-is-the-way ----------- diff --git a/fail2ban/version.py b/fail2ban/version.py index e5efbe77..078e47ef 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.11.1" +version = "1.0.1.dev1" def normVersion(): """ Returns fail2ban version in normalized machine-readable format""" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 745c080a..54b475e3 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "January 2020" "fail2ban-client v0.11.1" "User Commands" +.TH FAIL2BAN-CLIENT "1" "January 2020" "fail2ban-client v1.0.1.dev1" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.11.1 reads log file that contains password failure report +Fail2Ban v1.0.1.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-python.1 b/man/fail2ban-python.1 index 2c9746ee..5237f0fc 100644 --- a/man/fail2ban-python.1 +++ b/man/fail2ban-python.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-PYTHON "1" "January 2020" "fail2ban-python 0.11.1" "User Commands" +.TH FAIL2BAN-PYTHON "1" "January 2020" "fail2ban-python 1.0.1.1" "User Commands" .SH NAME fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used .SH DESCRIPTION diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index a703130c..d11fcf43 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "January 2020" "fail2ban-regex 0.11.1" "User Commands" +.TH FAIL2BAN-REGEX "1" "January 2020" "fail2ban-regex 1.0.1.dev1" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 418b46dd..ed8aa639 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "January 2020" "fail2ban-server v0.11.1" "User Commands" +.TH FAIL2BAN-SERVER "1" "January 2020" "fail2ban-server v1.0.1.dev1" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.11.1 reads log file that contains password failure report +Fail2Ban v1.0.1.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 6c32cb9e..7a6fa4e4 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "January 2020" "fail2ban-testcases 0.11.1" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "January 2020" "fail2ban-testcases 1.0.1.dev1" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 3f5c382a988bb21f48a0fe3a26c2433d04f7076f Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 14 Jan 2020 20:55:24 +0100 Subject: [PATCH 02/15] readme: amend to bump version (v.1.0.1 is master now) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e9f5c3a..fac3414d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.11.0.dev1 20??/??/?? + v1.0.1.dev1 20??/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -46,11 +46,11 @@ Optional: To install: - tar xvfj fail2ban-0.11.0.tar.bz2 - cd fail2ban-0.11.0 + tar xvfj fail2ban-1.0.1.tar.bz2 + cd fail2ban-1.0.1 sudo python setup.py install -Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, 0.11 +Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, master or 0.11 git clone https://github.com/fail2ban/fail2ban.git cd fail2ban @@ -89,11 +89,11 @@ fail2ban(1) and jail.conf(5) manpages for further references. Code status: ------------ -* travis-ci.org: [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.11)](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) +* travis-ci.org: [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=master)](https://travis-ci.org/fail2ban/fail2ban?branch=master) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.11)](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) -* coveralls.io: [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.11)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) +* coveralls.io: [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=master)](https://coveralls.io/github/fail2ban/fail2ban?branch=master) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.11)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) -* codecov.io: [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.11)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) +* codecov.io: [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=master)](https://codecov.io/gh/fail2ban/fail2ban/branch/master) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.11)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) Contact: -------- From ed4b5b8bdc8914e9d8dc68ca889f7c05ea194986 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Jan 2020 13:06:21 +0100 Subject: [PATCH 03/15] change actioncheck behavior (on error only, gh-488) --- fail2ban/server/action.py | 51 ++++++++++++++++++------------- fail2ban/tests/actionstestcase.py | 6 ++++ fail2ban/tests/actiontestcase.py | 46 ++++++++++++++++++++++++++-- fail2ban/tests/servertestcase.py | 21 ++++++++++--- 4 files changed, 95 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 5c817fc0..4df7217e 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -407,7 +407,7 @@ class CommandAction(ActionBase): cmd = self.replaceTag(tag, self._properties, conditional=('family='+family if family else ''), cache=self.__substCache) - if '<' not in cmd or not family: return cmd + if not family or '<' not in cmd: return cmd # replace family as dynamic tags, important - don't cache, no recursion and auto-escape here: cmd = self.replaceDynamicTags(cmd, {'family':family}) return cmd @@ -962,31 +962,38 @@ class CommandAction(ActionBase): except (KeyError, TypeError): family = '' - # invariant check: - if self.actioncheck: - # don't repair/restore if unban (no matter): - def _beforeRepair(): - if cmd == '' and not self._properties.get('actionrepair_on_unban'): - self._logSys.error("Invariant check failed. Unban is impossible.") + repcnt = 0 + while True: + + # got some error, do invariant check: + if repcnt and self.actioncheck: + # don't repair/restore if unban (no matter): + def _beforeRepair(): + if cmd == '' and not self._properties.get('actionrepair_on_unban'): + self._logSys.error("Invariant check failed. Unban is impossible.") + return False + return True + # check and repair if broken: + ret = self._invariantCheck(family, _beforeRepair, forceStart=(cmd != '')) + # if not sane (and not restored) return: + if ret != 1: return False - return True - # check and repair if broken: - ret = self._invariantCheck(family, _beforeRepair, forceStart=(cmd != '')) - # if not sane (and not restored) return: - if ret != 1: - return False - # Replace static fields - realCmd = self.replaceTag(cmd, self._properties, - conditional=('family='+family if family else ''), cache=self.__substCache) + # Replace static fields + realCmd = self.replaceTag(cmd, self._properties, + conditional=('family='+family if family else ''), cache=self.__substCache) - # Replace dynamical tags, important - don't cache, no recursion and auto-escape here - if aInfo is not None: - realCmd = self.replaceDynamicTags(realCmd, aInfo) - else: - realCmd = cmd + # Replace dynamical tags, important - don't cache, no recursion and auto-escape here + if aInfo is not None: + realCmd = self.replaceDynamicTags(realCmd, aInfo) + else: + realCmd = cmd - return self.executeCmd(realCmd, self.timeout) + # try execute command: + ret = self.executeCmd(realCmd, self.timeout) + repcnt += 1 + if ret or repcnt > 1: + return ret @staticmethod def executeCmd(realCmd, timeout=60, **kwargs): diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index a3c14a4b..b324b2c4 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -215,6 +215,9 @@ class ExecuteActions(LogCaptureTestCase): # flush for inet6 is intentionally "broken" here - test no unhandled except and invariant check: act['actionflush?family=inet6'] = act.actionflush + '; exit 1' act.actionstart_on_demand = True + # force errors via check in ban/unban: + act.actionban = " ; " + act.actionban + act.actionunban = " ; " + act.actionunban self.__actions.start() self.assertNotLogged("stdout: %r" % 'ip start') @@ -292,6 +295,9 @@ class ExecuteActions(LogCaptureTestCase): act['actionflush?family=inet6'] = act.actionflush + '; exit 1' act.actionstart_on_demand = True act.actionrepair_on_unban = True + # force errors via check in ban/unban: + act.actionban = " ; " + act.actionban + act.actionunban = " ; " + act.actionunban self.__actions.start() self.assertNotLogged("stdout: %r" % 'ip start') diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 1a00c040..2dab3ab1 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -305,8 +305,8 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(self.__action.actionstart, "touch '%s'" % tmp) self.__action.actionstop = "rm -f '%s'" % tmp self.assertEqual(self.__action.actionstop, "rm -f '%s'" % tmp) - self.__action.actionban = "echo -n" - self.assertEqual(self.__action.actionban, 'echo -n') + self.__action.actionban = " && echo -n" + self.assertEqual(self.__action.actionban, " && echo -n") self.__action.actioncheck = "[ -e '%s' ]" % tmp self.assertEqual(self.__action.actioncheck, "[ -e '%s' ]" % tmp) self.__action.actionunban = "true" @@ -316,6 +316,7 @@ class CommandActionTest(LogCaptureTestCase): self.assertNotLogged('returned') # no action was actually executed yet + # start on demand is false, so it should cause failure on first attempt of ban: self.__action.ban({'ip': None}) self.assertLogged('Invariant check failed') self.assertLogged('returned successfully') @@ -365,12 +366,51 @@ class CommandActionTest(LogCaptureTestCase): self.pruneLog('[phase 2]') self.__action.actionstart = "touch '%s'" % tmp self.__action.actionstop = "rm '%s'" % tmp - self.__action.actionban = """printf "%%%%b\n" >> '%s'""" % tmp + self.__action.actionban = """ && printf "%%%%b\n" >> '%s'""" % tmp self.__action.actioncheck = "[ -e '%s' ]" % tmp self.__action.ban({'ip': None}) self.assertLogged('Invariant check failed') self.assertNotLogged('Unable to restore environment') + @with_tmpdir + def testExecuteActionCheckOnBanFailure(self, tmp): + tmp += '/fail2ban.test' + self.__action.actionstart = "touch '%s'; echo 'started ...'" % tmp + self.__action.actionstop = "rm -f '%s'" % tmp + self.__action.actionban = "[ -e '%s' ] && echo 'banned '" % tmp + self.__action.actioncheck = "[ -e '%s' ] && echo 'check ok' || { echo 'check failed'; exit 1; }" % tmp + self.__action.actionrepair = "echo 'repair ...'; touch '%s'" % tmp + self.__action.actionstart_on_demand = False + self.__action.start() + # phase 1: with repair; + # phase 2: without repair (start/stop), not on demand; + # phase 3: without repair (start/stop), start on demand. + for i in (1, 2, 3): + self.pruneLog('[phase %s]' % i) + # 1st time with success ban: + self.__action.ban({'ip': '192.0.2.1'}) + self.assertLogged( + "stdout: %r" % 'banned 192.0.2.1', all=True) + self.assertNotLogged("Invariant check failed. Trying", + "stdout: %r" % 'check failed', + "stdout: %r" % ('repair ...' if self.__action.actionrepair else 'started ...'), + "stdout: %r" % 'check ok', all=True) + # force error in ban: + os.remove(tmp) + self.pruneLog() + # 2nd time with fail recognition, success repair, check and ban: + self.__action.ban({'ip': '192.0.2.2'}) + self.assertLogged("Invariant check failed. Trying", + "stdout: %r" % 'check failed', + "stdout: %r" % ('repair ...' if self.__action.actionrepair else 'started ...'), + "stdout: %r" % 'check ok', + "stdout: %r" % 'banned 192.0.2.2', all=True) + # repeat without repair (stop/start), herafter enable on demand: + if self.__action.actionrepair: + self.__action.actionrepair = "" + elif not self.__action.actionstart_on_demand: + self.__action.actionstart_on_demand = True + @with_tmpdir def testExecuteActionCheckRepairEnvironment(self, tmp): tmp += '/fail2ban.test' diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 55e72455..71347c7e 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -2030,25 +2030,38 @@ class ServerConfigReaderTests(LogCaptureTestCase): action.ban(aInfos['ipv4']) if tests.get('ip4-start'): self.assertLogged(*tests.get('*-start', ())+tests['ip4-start'], all=True) if tests.get('ip6-start'): self.assertNotLogged(*tests['ip6-start'], all=True) - self.assertLogged(*tests.get('ip4-check',())+tests['ip4-ban'], all=True) + self.assertLogged(*tests['ip4-ban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test unban ip4 : self.pruneLog('# === unban ipv4 ===') action.unban(aInfos['ipv4']) - self.assertLogged(*tests.get('ip4-check',())+tests['ip4-unban'], all=True) + self.assertLogged(*tests['ip4-unban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test ban ip6 : self.pruneLog('# === ban ipv6 ===') action.ban(aInfos['ipv6']) if tests.get('ip6-start'): self.assertLogged(*tests.get('*-start', ())+tests['ip6-start'], all=True) if tests.get('ip4-start'): self.assertNotLogged(*tests['ip4-start'], all=True) - self.assertLogged(*tests.get('ip6-check',())+tests['ip6-ban'], all=True) + self.assertLogged(*tests['ip6-ban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test unban ip6 : self.pruneLog('# === unban ipv6 ===') action.unban(aInfos['ipv6']) - self.assertLogged(*tests.get('ip6-check',())+tests['ip6-unban'], all=True) + self.assertLogged(*tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) + # test invariant check (normally on demand in error case only): + if tests.get('ip4-check'): + self.pruneLog('# === check ipv4 ===') + action._invariantCheck(aInfos['ipv4']['family']) + self.assertLogged(*tests['ip4-check'], all=True) + if tests.get('ip6-check') and tests['ip6-check'] != tests['ip4-check']: + self.assertNotLogged(*tests['ip6-check'], all=True) + if tests.get('ip6-check'): + self.pruneLog('# === check ipv6 ===') + action._invariantCheck(aInfos['ipv6']['family']) + self.assertLogged(*tests['ip6-check'], all=True) + if tests.get('ip4-check') and tests['ip4-check'] != tests['ip6-check']: + self.assertNotLogged(*tests['ip4-check'], all=True) # test flush for actions should supported this: if tests.get('flush'): self.pruneLog('# === flush ===') From 8cbc1e0ebb91142a34756fd2f4416b08d773deaa Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Jan 2020 16:51:27 +0100 Subject: [PATCH 04/15] ChangeLog (change actioncheck behavior) --- ChangeLog | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3f9c11b1..73e0e047 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,11 +9,20 @@ Fail2Ban: Changelog ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition ----------- +### Compatibility: +* to v.0.11: + - due to change of `actioncheck` behavior (gh-488), some actions can be incompatible as regards + the invariant check, if `actionban` or `actionunban` would not throw an error (exit code + different from 0) in case of unsane environment. + ### Fixes ### New Features ### Enhancements +* `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair + of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. + if ban or unban operations are exiting with other code as 0) ver. 0.11.1 (2020/01/11) - this-is-the-way From 3c83c19070b8e79e2d4392071d02b4ea99643d87 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sat, 6 Jun 2020 19:51:46 +0200 Subject: [PATCH 05/15] Added filter nginx-bad-request --- config/filter.d/nginx-bad-request.conf | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 config/filter.d/nginx-bad-request.conf diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf new file mode 100644 index 00000000..ea26d56a --- /dev/null +++ b/config/filter.d/nginx-bad-request.conf @@ -0,0 +1,13 @@ +# Fail2Ban filter to match bad requests to nginx +# + +[Definition] + +# The request often doesn't contain a method, only some encoded garbage +failregex = ^ \- \S+ \[\] \".+\" 400 .+$ + +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + ^[^\[]*\[({DATE}) + {^LN-BEG} + +# Author: Jan Przybylak From d7ef5d166db58ec402408fa97c0b291906c3d8c9 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Thu, 11 Jun 2020 16:44:48 +0200 Subject: [PATCH 06/15] Removed vulnerable catchall & anchor --- config/filter.d/nginx-bad-request.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index ea26d56a..03721f86 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -4,7 +4,7 @@ [Definition] # The request often doesn't contain a method, only some encoded garbage -failregex = ^ \- \S+ \[\] \".+\" 400 .+$ +failregex = ^ \- \S+ \[\] \"[^\"]+\" 400 datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? ^[^\[]*\[({DATE}) From a5ab4406d8b7d34da8e653d8d5debd2c0735b484 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sun, 21 Jun 2020 18:24:09 +0200 Subject: [PATCH 07/15] Removed unnecessary escape sequence This commit also contains changes to match requests that are 100% empty (by using "*" instead of "+" in the regex) --- config/filter.d/nginx-bad-request.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf index 03721f86..2b8f5ab6 100644 --- a/config/filter.d/nginx-bad-request.conf +++ b/config/filter.d/nginx-bad-request.conf @@ -4,7 +4,8 @@ [Definition] # The request often doesn't contain a method, only some encoded garbage -failregex = ^ \- \S+ \[\] \"[^\"]+\" 400 +# This will also match requests that are entirely empty +failregex = ^ - \S+ \[\] "[^"]*" 400 datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? ^[^\[]*\[({DATE}) From 56fefe9240a6cc06650b91743339451caf6aa2f1 Mon Sep 17 00:00:00 2001 From: Jan Przybylak Date: Sun, 21 Jun 2020 18:25:27 +0200 Subject: [PATCH 08/15] Added test file "nginx-bad-request" I tested with `./fail2ban-testcases testSampleRegex`, which did not return any errors. --- fail2ban/tests/files/logs/nginx-bad-request | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 fail2ban/tests/files/logs/nginx-bad-request diff --git a/fail2ban/tests/files/logs/nginx-bad-request b/fail2ban/tests/files/logs/nginx-bad-request new file mode 100644 index 00000000..a9ff6497 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-bad-request @@ -0,0 +1,23 @@ +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - root [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "\x03\x00\x00/*\xE0\x00\x00\x00\x00\x00Cookie: mstshash=Administr" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "GET //admin/pma/scripts/setup.php HTTP/1.1" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:54:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:54:28 +0100] "HELP" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:55:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:55:28 +0100] "batman" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T01:17:07", "match": true , "host": "7.8.9.10" } +7.8.9.10 - root [20/Jan/2015:01:17:07 +0100] "CONNECT 123.123.123.123 HTTP/1.1" 400 162 "-" "-" "-" + +# failJSON: { "time": "2014-12-12T22:59:02", "match": true , "host": "2.5.2.5" } +2.5.2.5 - tomcat [12/Dec/2014:22:59:02 +0100] "GET /cgi-bin/tools/tools.pl HTTP/1.1" 400 162 "-" "-" "-" \ No newline at end of file From fd25c4cbb813db2562e5a2e54d4510eac291a897 Mon Sep 17 00:00:00 2001 From: TorontoMedia <10255876+TorontoMedia@users.noreply.github.com> Date: Sun, 28 Jun 2020 12:58:41 -0400 Subject: [PATCH 09/15] Remove duplicate method and rename invalid parameter --- fail2ban/server/failmanager.py | 4 ---- fail2ban/server/observer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index bc09c0e2..97b4c7a4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c19549ba..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: From 9d77fb2b4c5f4471e1abef0974c70c1ff622ff75 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 3 Nov 2020 13:15:42 +0100 Subject: [PATCH 10/15] 1st try of GH actions flow (CI only, no coverage atm) --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..154a75ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + python -m pip install --upgrade pip + pip install 2to3 + fi + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 753fff9c15387d62e32bc9794cb7a4714b9dd359 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 23 Nov 2020 18:38:41 +0100 Subject: [PATCH 11/15] amend to #2750, add jail for new filter nginx-bad-request --- config/jail.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index e6961a18..28f259a0 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -392,7 +392,10 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_error_log)s -maxretry = 2 + +[nginx-bad-request] +port = http,https +logpath = %(nginx_access_log)s # Ban attackers that try to use PHP's URL-fopen() functionality From 0f27a8add82dd75c9cb984505732b56c1f625313 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 24 Nov 2020 19:28:46 +0100 Subject: [PATCH 12/15] datedetector, strptime: token `%Z` must recognize zone abbreviation `Z` (GMT/UTC) also, similar to `%z`; more test cases added. --- fail2ban/server/strptime.py | 8 ++------ fail2ban/tests/datedetectortestcase.py | 3 +++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 1464a96d..5fcfb959 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -234,16 +234,12 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): week_of_year = int(val) # U starts week on Sunday, W - on Monday week_of_year_start = 6 if key == 'U' else 0 - elif key == 'z': + elif key in ('z', 'Z'): z = val if z in ("Z", "UTC", "GMT"): tzoffset = 0 - else: + elif key == 'z': tzoffset = zone2offset(z, 0); # currently offset-based only - elif key == 'Z': - z = val - if z in ("UTC", "GMT"): - tzoffset = 0 # Fail2Ban will assume it's this year assume_year = False diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index d6370fc4..ee126f72 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -516,6 +516,9 @@ class CustomDateFormatsTest(unittest.TestCase): (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 UTC] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 UTC] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 Z] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 +0000] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 Z] server ..."), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: From b8e8a87ee93985d36c8f53bf5089650b34c874e1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 24 Nov 2020 20:21:49 +0100 Subject: [PATCH 13/15] small amend to 0f27a8add82dd75c9cb984505732b56c1f625313 - datedetector, strptime: token `%Z` recognizes all known zone abbreviation besides Z, GMT, UTC correctly, if it is matching (`%z` remains unchanged for backwards-compatibility, see comment in code); test cases fixed (PDT zone will be found now). --- fail2ban/server/strptime.py | 2 +- fail2ban/tests/files/logs/monit | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 5fcfb959..7f11402a 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -238,7 +238,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): z = val if z in ("Z", "UTC", "GMT"): tzoffset = 0 - elif key == 'z': + else: tzoffset = zone2offset(z, 0); # currently offset-based only # Fail2Ban will assume it's this year diff --git a/fail2ban/tests/files/logs/monit b/fail2ban/tests/files/logs/monit index 8dbddaf6..36f1c1e4 100644 --- a/fail2ban/tests/files/logs/monit +++ b/fail2ban/tests/files/logs/monit @@ -1,7 +1,7 @@ # Previous version -- -# failJSON: { "time": "2005-04-16T21:05:29", "match": true , "host": "69.93.127.111" } +# failJSON: { "time": "2005-04-17T06:05:29", "match": true , "host": "69.93.127.111" } [PDT Apr 16 21:05:29] error : Warning: Client '69.93.127.111' supplied unknown user 'foo' accessing monit httpd -# failJSON: { "time": "2005-04-16T20:59:33", "match": true , "host": "97.113.189.111" } +# failJSON: { "time": "2005-04-17T05:59:33", "match": true , "host": "97.113.189.111" } [PDT Apr 16 20:59:33] error : Warning: Client '97.113.189.111' supplied wrong password for user 'admin' accessing monit httpd # Current version -- corresponding "https://bitbucket.org/tildeslash/monit/src/6905335aa903d425cae732cab766bd88ea5f2d1d/src/http/processor.c?at=master&fileviewer=file-view-default#processor.c-728" From 3bb19ecc9d9fe18c3b5ac6a25844d17839543111 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Wed, 23 Dec 2020 12:20:42 +1100 Subject: [PATCH 14/15] docs: fix simple typo, litle -> little There is a small typo in fail2ban/client/fail2banclient.py, fail2ban/server/observer.py. Should read `little` rather than `litle`. --- fail2ban/client/fail2banclient.py | 2 +- fail2ban/server/observer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 6ea18fda..a7053034 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -230,7 +230,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.log(5, ' client phase %s', phase) if not stream: return False - # wait a litle bit for phase "start-ready" before enter active waiting: + # wait a little bit for phase "start-ready" before enter active waiting: if phase is not None: Utils.wait_for(lambda: phase.get('start-ready', None) is not None, 0.5, 0.001) phase['configure'] = (True if stream else False) diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index f5ba20d9..996eec4a 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -232,7 +232,7 @@ class ObserverThread(JailThread): if self._paused: continue else: - ## notify event deleted (shutdown) - just sleep a litle bit (waiting for shutdown events, prevent high cpu usage) + ## notify event deleted (shutdown) - just sleep a little bit (waiting for shutdown events, prevent high cpu usage) time.sleep(ObserverThread.DEFAULT_SLEEP_INTERVAL) ## stop by shutdown and empty queue : if not self.is_full: From ba7daef86c521ceda4ee48b5f221485dd98a13ad Mon Sep 17 00:00:00 2001 From: defanor Date: Thu, 24 Dec 2020 06:55:01 +0300 Subject: [PATCH 15/15] Handle postscreen's PREGREET and HANGUP messages Provoking those seems to be a popular activity among spammers. --- config/filter.d/postfix.conf | 4 +++- fail2ban/tests/files/logs/postfix | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index fb690fb0..69b4ab48 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -37,7 +37,9 @@ mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unava mdpr-more = %(mdpr-normal)s mdre-more = %(mdre-normal)s -mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))) +# Includes some of the log messages described in +# . +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+) mdre-ddos = ^from [^[]*\[\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 6e2dc460..9f74e155 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -151,6 +151,11 @@ Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[1 # failJSON: { "time": "2005-02-18T09:48:04", "match": true , "host": "192.0.2.23" } Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[192.0.2.23] +# failJSON: { "time": "2004-12-23T19:39:13", "match": true , "host": "192.0.2.2" } +Dec 23 19:39:13 xxx postfix/postscreen[21057]: PREGREET 14 after 0.08 from [192.0.2.2]:59415: EHLO ylmf-pc\r\n +# failJSON: { "time": "2004-12-24T00:54:36", "match": true , "host": "192.0.2.3" } +Dec 24 00:54:36 xxx postfix/postscreen[22515]: HANGUP after 16 from [192.0.2.3]:48119 in tests after SMTP handshake + # filterOptions: [{}, {"mode": "ddos"}, {"mode": "aggressive"}] # failJSON: { "match": false, "desc": "don't affect lawful data (sporadical connection aborts within DATA-phase, see gh-1813 for discussion)" } Feb 18 09:50:05 xxx postfix/smtpd[42]: lost connection after DATA from good-host.example.com[192.0.2.10]