From d0d1f8c362eb927a90abd6c86bab31a8a7717a79 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 26 Aug 2020 16:54:18 +0200
Subject: [PATCH 01/16] improve result for get/set prefregex

---
 fail2ban/server/transmitter.py   | 6 ++++--
 fail2ban/tests/servertestcase.py | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index aff9071c..10cfd163 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -277,7 +277,8 @@ class Transmitter:
 			value = command[2]
 			self.__server.setPrefRegex(name, value)
 			if self.__quiet: return
-			return self.__server.getPrefRegex(name)
+			v = self.__server.getPrefRegex(name)
+			return v.getRegex() if v else ""
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value, multiple=multiple)
@@ -446,7 +447,8 @@ class Transmitter:
 		elif command[1] == "ignorecache":
 			return self.__server.getIgnoreCache(name)
 		elif command[1] == "prefregex":
-			return self.__server.getPrefRegex(name)
+			v = self.__server.getPrefRegex(name)
+			return v.getRegex() if v else ""
 		elif command[1] == "failregex":
 			return self.__server.getFailRegex(name)
 		elif command[1] == "ignoreregex":
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index f1b667b1..3b2552dd 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -513,6 +513,9 @@ class Transmitter(TransmitterBase):
 			jail=self.jailName)
 		self.setGetTest("ignorecache", '', None, jail=self.jailName)
 
+	def testJailPrefRegex(self):
+		self.setGetTest("prefregex", "^Test", jail=self.jailName)
+
 	def testJailRegex(self):
 		self.jailAddDelRegexTest("failregex",
 			[

From 07fa9f29129f53924f6850f01adeb8fb7c699060 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 27 Aug 2020 17:04:19 +0200
Subject: [PATCH 02/16] fixes gh-2787: allow to match `did not issue
 MAIL/EXPN/VRFY/ETRN during connection` non-anchored with extra mode (default
 names may deviate); additionally provides common addr-tag for IPv4/IPv6
 (`(?:IPv6:<IP6>|<IP4>)`) and test-coverage for IPv6

---
 config/filter.d/sendmail-reject.conf      | 13 +++++++------
 fail2ban/tests/files/logs/sendmail-reject |  4 ++++
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf
index ca171915..e8b766c5 100644
--- a/config/filter.d/sendmail-reject.conf
+++ b/config/filter.d/sendmail-reject.conf
@@ -21,19 +21,20 @@ before = common.conf
 
 _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:IPv6:<IP6>|<IP4>)
 
 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
-            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=(?:IPv6:<IP6>|<IP4>), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^rejecting commands from (\S* )?\[(?:IPv6:<IP6>|<IP4>)\] due to pre-greeting traffic after \d+ seconds$
-            ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
+            ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$
+            ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
             ^<[^@]+@[^>]+>\.\.\. No such user here$
-            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:<IP6>|<IP4>)\]$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$
 
 mdre-normal =
 
-mdre-extra = ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$
+mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection
 
 mdre-aggressive = %(mdre-extra)s
 
diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject
index f69e4531..99c1877c 100644
--- a/fail2ban/tests/files/logs/sendmail-reject
+++ b/fail2ban/tests/files/logs/sendmail-reject
@@ -103,3 +103,7 @@ Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52
 
 # failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" }
 Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA
+# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" }
+Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4
+# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" }
+Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6
\ No newline at end of file

From 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 27 Aug 2020 17:44:25 +0200
Subject: [PATCH 03/16] filter `sendmail-auth` extended to follow new
 authentication failure message introduced in sendmail 8.16.1,
 AUTH_FAIL_LOG_USER (gh-2757)

---
 config/filter.d/sendmail-auth.conf      | 5 +++--
 fail2ban/tests/files/logs/sendmail-auth | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf
index 14995fed..c15a2e0c 100644
--- a/config/filter.d/sendmail-auth.conf
+++ b/config/filter.d/sendmail-auth.conf
@@ -9,10 +9,11 @@ before = common.conf
 
 _daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
+addr = (?:IPv6:<IP6>|<IP4>)
 
 # "w{14,20}" will give support for IDs from 14 up to 20 characters long
-failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
-
+failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+            ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=<F-USER>(?:\S+|.*?)</F-USER>, relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$
 ignoreregex =
 
 journalmatch = _SYSTEMD_UNIT=sendmail.service
diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth
index a7ddd6f8..93bf0b14 100644
--- a/fail2ban/tests/files/logs/sendmail-auth
+++ b/fail2ban/tests/files/logs/sendmail-auth
@@ -17,3 +17,8 @@ Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SM
 
 # failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" }
 Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5
+
+# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" }
+Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged)
+# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" }
+Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged)

From db1f3477cc415e5b89191b25f36406613012fc6d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 27 Aug 2020 18:07:42 +0200
Subject: [PATCH 04/16] amend to 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a:
 sendmail-auth has 2 failregex now, so rewritten with prefregex

---
 config/filter.d/sendmail-auth.conf | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf
index c15a2e0c..84fcbdda 100644
--- a/config/filter.d/sendmail-auth.conf
+++ b/config/filter.d/sendmail-auth.conf
@@ -8,12 +8,14 @@ before = common.conf
 [Definition]
 
 _daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
+# "\w{14,20}" will give support for IDs from 14 up to 20 characters long
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
 addr = (?:IPv6:<IP6>|<IP4>)
 
-# "w{14,20}" will give support for IDs from 14 up to 20 characters long
-failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
-            ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=<F-USER>(?:\S+|.*?)</F-USER>, relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$
+prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+            ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=<F-USER>(?:\S+|.*?)</F-USER>, relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$
 ignoreregex =
 
 journalmatch = _SYSTEMD_UNIT=sendmail.service

From 17a6ba44b3fc9e60b6905ea2694a56f2c58c783b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 28 Aug 2020 13:26:26 +0200
Subject: [PATCH 05/16] fail2ban-regex: speedup formatted output (bypass
 unneeded stats creation); fail2ban-regex: extended with prefregex statistic

---
 ChangeLog                        |  3 +++
 fail2ban/client/fail2banregex.py | 43 ++++++++++++++++++++++++++------
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cacfdee7..c7f9c759 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -62,6 +62,9 @@ ver. 0.10.6-dev (20??/??/??) - development edition
 * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749)
 
 ### New Features and Enhancements
+* fail2ban-regex:
+  - speedup formatted output (bypass unneeded stats creation)
+  - extended with prefregex statistic
 * new filter and jail for GitLab recognizing failed application logins (gh-2689)
 * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
 * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 98fd9799..1adf9761 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -252,6 +252,8 @@ class Fail2banRegex(object):
 
 		self.share_config=dict()
 		self._filter = Filter(None)
+		self._prefREMatched = 0
+		self._prefREGroups = list()
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._time_elapsed = None
@@ -453,19 +455,33 @@ class Fail2banRegex(object):
 			lines = []
 			ret = []
 			for match in found:
-				# Append True/False flag depending if line was matched by
-				# more than one regex
-				match.append(len(ret)>1)
-				regex = self._failregex[match[0]]
-				regex.inc()
-				regex.appendIP(match)
+				if not self._opts.out:
+					# Append True/False flag depending if line was matched by
+					# more than one regex
+					match.append(len(ret)>1)
+					regex = self._failregex[match[0]]
+					regex.inc()
+					regex.appendIP(match)
 				if not match[3].get('nofail'):
 					ret.append(match)
 				else:
 					is_ignored = True
+			if self._opts.out: # (formated) output - don't need stats:
+				return None, ret, None
+			# prefregex stats:
+			if self._filter.prefRegex:
+				pre = self._filter.prefRegex
+				if pre.hasMatched():
+					self._prefREMatched += 1
+					if self._verbose:
+						if len(self._prefREGroups) < self._maxlines:
+							self._prefREGroups.append(pre.getGroups())
+						else:
+							if len(self._prefREGroups) == self._maxlines:
+								self._prefREGroups.append('...')
 		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
-			return False
+			return None, 0, None
 		if self._filter.getMaxLines() > 1:
 			for bufLine in orgLineBuffer[int(fullBuffer):]:
 				if bufLine not in self._filter._Filter__lineBuffer:
@@ -651,7 +667,18 @@ class Fail2banRegex(object):
 			pprint_list(out, " #) [# of hits] regular expression")
 			return total
 
-		# Print title
+		# Print prefregex:
+		if self._filter.prefRegex:
+			#self._filter.prefRegex.hasMatched()
+			pre = self._filter.prefRegex 
+			out = [pre.getRegex()]
+			if self._verbose:
+				for grp in self._prefREGroups:
+					out.append("    %s" % (grp,))
+			output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) )
+			pprint_list(out)
+
+		# Print regex's:
 		total = print_failregexes("Failregex", self._failregex)
 		_ = print_failregexes("Ignoreregex", self._ignoreregex)
 

From a3a148078e1875fd7b33daa1f07e0bb0c2000342 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 28 Aug 2020 14:12:57 +0200
Subject: [PATCH 06/16] fail2ban-regex: more informative output for
 `datepattern` (e. g. set from filter) - pattern : description, example: Use  
    datepattern : ^%Y-%m-%d %H:%M:%S : ^Year-Month-Day 24hour:Minute:Second

---
 ChangeLog                        | 1 +
 fail2ban/client/fail2banregex.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c7f9c759..fabfed68 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -65,6 +65,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition
 * fail2ban-regex:
   - speedup formatted output (bypass unneeded stats creation)
   - extended with prefregex statistic
+  - more informative output for `datepattern` (e. g. set from filter) - pattern : description
 * new filter and jail for GitLab recognizing failed application logins (gh-2689)
 * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
 * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 1adf9761..d0fe55dc 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -294,8 +294,8 @@ class Fail2banRegex(object):
 			self._filter.setDatePattern(pattern)
 			self._datepattern_set = True
 			if pattern is not None:
-				self.output( "Use      datepattern : %s" % (
-					self._filter.getDatePattern()[1], ) )
+				self.output( "Use      datepattern : %s : %s" % (
+					pattern, self._filter.getDatePattern()[1], ) )
 
 	def setMaxLines(self, v):
 		if not self._maxlines_set:

From f09ba1b3993c804aaa279b4542fa8ffd87c4dd53 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 2 Sep 2020 19:59:27 +0200
Subject: [PATCH 07/16] action in jail-config extended to consider space as
 separator now (splitWithOptions separates by space between mains words, but
 not in options), so defining `action = a b` would specify 2 actions `a` and
 `b`; it is additionally more precise now (see fixed typo with closed bracket
 `]` instead of comma in testServerReloadTest)

---
 ChangeLog                                |  2 ++
 fail2ban/helpers.py                      |  2 +-
 fail2ban/tests/clientreadertestcase.py   | 11 +++++++++++
 fail2ban/tests/fail2banclienttestcase.py |  2 +-
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index fabfed68..8e9b7612 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -66,6 +66,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition
   - speedup formatted output (bypass unneeded stats creation)
   - extended with prefregex statistic
   - more informative output for `datepattern` (e. g. set from filter) - pattern : description
+* parsing of action in jail-configs considers space between action-names as separator also
+  (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b`
 * new filter and jail for GitLab recognizing failed application logins (gh-2689)
 * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631)
 * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex;
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index dc7852ae..f381576e 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -373,7 +373,7 @@ OPTION_EXTRACT_CRE = re.compile(
 	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
 # split by new-line considering possible new-lines within options [...]:
 OPTION_SPLIT_CRE = re.compile(
-	r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL)
+	r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL)
 
 def extractOptions(option):
 	match = OPTION_CRE.match(option)
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index bb42b7a0..2cfaff77 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -264,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase):
 	def __init__(self, *args, **kwargs):
 		super(JailReaderTest, self).__init__(*args, **kwargs)
 
+	def testSplitWithOptions(self):
+		# covering all separators - new-line and spaces:
+		for sep in ('\n', '\t', ' '):
+			self.assertEqual(splitWithOptions('a%sb' % (sep,)),           ['a',           'b'])
+			self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)),      ['a[x=y]',      'b'])
+			self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)),  ['a[x="y z"]',  'b'])
+			self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b'])
+			self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b'])
+
 	def testIncorrectJail(self):
 		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG)
 		self.assertRaises(ValueError, jail.read)
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index bbd6964a..03b1d7ce 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -856,7 +856,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']"
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>',"
 										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test2log,

From ed20d457b21d298cf8a8569ab2eb09546e8065c4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 2 Sep 2020 20:14:31 +0200
Subject: [PATCH 08/16] jail.conf: removed action parameter `name` that set on
 jail-name (`name=%(__name__)s` is default in action reader)

---
 config/jail.conf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 40827707..8fbd23a1 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -171,16 +171,16 @@ banaction = iptables-multiport
 banaction_allports = iptables-allports
 
 # The simplest action to take: ban only
-action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
+action_ = %(banaction)s[bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
 
 # ban & send an e-mail with whois report to the destemail.
 action_mw = %(action_)s
-            %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
+            %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
 
 # ban & send an e-mail with whois report and relevant log lines
 # to the destemail.
 action_mwl = %(action_)s
-             %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+             %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
 
 # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action
 #
@@ -192,7 +192,7 @@ action_xarf = %(action_)s
 # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines
 # to the destemail.
 action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
-                %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
+                %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
 # 

From a038fd5dfe8cb0714472833604735b83462a217d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 3 Sep 2020 16:41:23 +0200
Subject: [PATCH 09/16] `action.d/firewallcmd-*.conf` (multiport only): fixed
 port range selector, replacing `:` with `-`; small optimizations on
 `firewallcmd-rich-rules.conf` and `firewallcmd-rich-logging.conf` simplifying
 both and provide a dependency (rich-logging is a derivative of rich-rules);
 closes gh-2821

---
 ChangeLog                                     |  1 +
 config/action.d/firewallcmd-ipset.conf        |  2 +-
 config/action.d/firewallcmd-multiport.conf    |  4 +-
 config/action.d/firewallcmd-new.conf          |  4 +-
 config/action.d/firewallcmd-rich-logging.conf | 30 ++----------
 config/action.d/firewallcmd-rich-rules.conf   |  8 ++--
 fail2ban/tests/servertestcase.py              | 48 +++++++++++++++----
 7 files changed, 55 insertions(+), 42 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e9b7612..361b81d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -51,6 +51,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition
 * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing
    with `jq`, gh-2140, gh-2656)
 * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763)
+* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821)
 * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`
   should be interpolated in definition section (inside the filter-config, gh-2650)
 * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697)
diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 42513933..66358f23 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -69,7 +69,7 @@ allports = -p <protocol>
 # Option: multiport
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config:  banaction = firewallcmd-ipset[actiontype=<multiport>]
-multiport = -p <protocol> -m multiport --dports <port>
+multiport = -p <protocol> -m multiport --dports "$(echo '<port>' | sed s/:/-/g)"
 
 ipmset = f2b-<name>
 familyopt =
diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf
index 81540e5b..0c401f1b 100644
--- a/config/action.d/firewallcmd-multiport.conf
+++ b/config/action.d/firewallcmd-multiport.conf
@@ -11,9 +11,9 @@ before = firewallcmd-common.conf
 
 actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
               firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports "$(echo '<port>' | sed s/:/-/g)" -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports "$(echo '<port>' | sed s/:/-/g)" -j f2b-<name>
              firewall-cmd --direct --remove-rules <family> filter f2b-<name>
              firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf
index b06f5ccd..7b08603c 100644
--- a/config/action.d/firewallcmd-new.conf
+++ b/config/action.d/firewallcmd-new.conf
@@ -10,9 +10,9 @@ before = firewallcmd-common.conf
 
 actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
               firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports "$(echo '<port>' | sed s/:/-/g)" -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports "$(echo '<port>' | sed s/:/-/g)" -j f2b-<name>
              firewall-cmd --direct --remove-rules <family> filter f2b-<name>
              firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf
index badfee83..21e45087 100644
--- a/config/action.d/firewallcmd-rich-logging.conf
+++ b/config/action.d/firewallcmd-rich-logging.conf
@@ -1,6 +1,6 @@
 # Fail2Ban configuration file
 #
-# Author: Donald Yandt 
+# Authors: Donald Yandt, Sergey G. Brester
 # 
 # Because of the rich rule commands requires firewalld-0.3.1+
 # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not
@@ -10,36 +10,15 @@
 #
 # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/<zone>.xml that can be shared and parsed easliy
 #
-# Example commands to view rules:
-# firewall-cmd [--zone=<zone>] --list-rich-rules
-# firewall-cmd [--zone=<zone>] --list-all
-# firewall-cmd [--zone=zone] --query-rich-rule='rule'
+# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters.
 
 [INCLUDES]
 
-before = firewallcmd-common.conf
+before = firewallcmd-rich-rules.conf
 
 [Definition]
 
-actionstart = 
-
-actionstop = 
-
-actioncheck = 
-
-# you can also use zones and/or service names. 
-#
-# zone example: 
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# service name example:
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
-#
-# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
-
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
-	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
+rich-suffix = log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>
 
 [Init]
 
@@ -48,4 +27,3 @@ level = info
 
 # log rate per minute
 rate = 1
-
diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf
index bed71797..803d7d12 100644
--- a/config/action.d/firewallcmd-rich-rules.conf
+++ b/config/action.d/firewallcmd-rich-rules.conf
@@ -35,8 +35,10 @@ actioncheck =
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
+fwcmd_rich_rule = rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' %(rich-suffix)s
+
+actionban = ports="$(echo '<port>' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
-
+actionunban = ports="$(echo '<port>' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done
 
+rich-suffix = <rich-blocktype>
\ No newline at end of file
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 3b2552dd..eaf1b346 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1780,18 +1780,18 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				'ip4-start': (
 					"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
 					"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
-					"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					"""`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""",
 				), 
 				'ip6-start': (
 					"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`",
 					"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
-					"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					"""`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""",
 				),
 				'stop': (
-					"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					"""`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""",
 					"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`",
 					"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`",
-					"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					"""`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""",
 					"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`",
 					"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`",
 				),
@@ -1859,21 +1859,21 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
 				'ip4-start': (
 					"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `",
-					"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+					"""`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""",
 				), 
 				'ip6-start': (
 					"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`",
-					"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					"""`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""",
 				),
 				'flush': (
 					"`ipset flush f2b-j-w-fwcmd-ipset`",
 					"`ipset flush f2b-j-w-fwcmd-ipset6`",
 				),
 				'stop': (
-					"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+					"""`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""",
 					"`ipset flush f2b-j-w-fwcmd-ipset`",
 					"`ipset destroy f2b-j-w-fwcmd-ipset`",
-					"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					"""`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""",
 					"`ipset flush f2b-j-w-fwcmd-ipset6`",
 					"`ipset destroy f2b-j-w-fwcmd-ipset6`",
 				),
@@ -1926,6 +1926,38 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`",
 				),					
 			}),
+			# firewallcmd-rich-rules --
+			('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', {
+				'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',),
+				'ip4-ban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""",
+				),
+				'ip4-unban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""",
+				),
+				'ip6-ban': (
+					""" `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""",
+				),
+				'ip6-unban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""",
+				),					
+			}),
+			# firewallcmd-rich-logging --
+			('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', {
+				'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',),
+				'ip4-ban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""",
+				),
+				'ip4-unban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""",
+				),
+				'ip6-ban': (
+					""" `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""",
+				),
+				'ip6-unban': (
+					"""`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""",
+				),					
+			}),
 		)
 		server = TestServer()
 		transm = server._Server__transm

From f555ff45e99c804a3fa48a2db1af56fa9200da0a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 7 Sep 2020 19:08:52 +0200
Subject: [PATCH 10/16] attempt to speedup ban- and fail-manager (e. g.
 fail2ban-client status, see gh-2819), remove unneeded lock (GIL is enough
 here)

---
 fail2ban/server/banmanager.py  | 20 +++++++-------------
 fail2ban/server/failmanager.py |  9 +++------
 2 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 479ba26f..fe38dec6 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -66,8 +66,7 @@ class BanManager:
 	# @param value the time
 	
 	def setBanTime(self, value):
-		with self.__lock:
-			self.__banTime = int(value)
+		self.__banTime = int(value)
 	
 	##
 	# Get the ban time.
@@ -76,8 +75,7 @@ class BanManager:
 	# @return the time
 	
 	def getBanTime(self):
-		with self.__lock:
-			return self.__banTime
+		return self.__banTime
 	
 	##
 	# Set the total number of banned address.
@@ -85,8 +83,7 @@ class BanManager:
 	# @param value total number
 	
 	def setBanTotal(self, value):
-		with self.__lock:
-			self.__banTotal = value
+		self.__banTotal = value
 	
 	##
 	# Get the total number of banned address.
@@ -94,8 +91,7 @@ class BanManager:
 	# @return the total number
 	
 	def getBanTotal(self):
-		with self.__lock:
-			return self.__banTotal
+		return self.__banTotal
 
 	##
 	# Returns a copy of the IP list.
@@ -103,8 +99,7 @@ class BanManager:
 	# @return IP list
 	
 	def getBanList(self):
-		with self.__lock:
-			return list(self.__banList.keys())
+		return list(self.__banList.keys())
 
 	##
 	# Returns a iterator to ban list (used in reload, so idle).
@@ -112,9 +107,8 @@ class BanManager:
 	# @return ban list iterator
 	
 	def __iter__(self):
-		# ensure iterator is safe (traverse over the list in snapshot created within lock):
-		with self.__lock:
-			return iter(list(self.__banList.values()))
+		# ensure iterator is safe - traverse over the list in snapshot created within lock (GIL):
+		return iter(list(self.__banList.values()))
 
 	##
 	# Returns normalized value
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 3458aed5..3e81e8b5 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -47,12 +47,10 @@ class FailManager:
 		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
-		with self.__lock:
-			self.__failTotal = value
+		self.__failTotal = value
 		
 	def getFailTotal(self):
-		with self.__lock:
-			return self.__failTotal
+		return self.__failTotal
 	
 	def getFailCount(self):
 		# may be slow on large list of failures, should be used for test purposes only...
@@ -123,8 +121,7 @@ class FailManager:
 		return attempts
 	
 	def size(self):
-		with self.__lock:
-			return len(self.__failList)
+		return len(self.__failList)
 	
 	def cleanup(self, time):
 		with self.__lock:

From 5abc4ba4ae280b6ec89c6bffc9dd2140d0d56dc4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 7 Sep 2020 22:11:51 +0200
Subject: [PATCH 11/16] amend to 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c
 (#2758): better reaction on broken pipe (on long output), don't close stdout
 explicitly (allows usage of modules like cProfile, which outputs result on
 exit), just flush it before exit.

---
 fail2ban/client/fail2bancmdline.py | 24 ++++++++++++++++--------
 fail2ban/helpers.py                |  5 +++--
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 8936e03f..03683cad 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -27,13 +27,17 @@ import sys
 
 from ..version import version, normVersion
 from ..protocol import printFormatted
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError
 
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
 def output(s): # pragma: no cover
-	print(s)
+	try:
+		print(s)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno != 32: # closed / broken pipe
+			raise
 
 # Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf),
 CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket")
@@ -310,12 +314,16 @@ class Fail2banCmdLine():
 	def _exit(code=0):
 		# implicit flush without to produce broken pipe error (32):
 		sys.stderr.close()
-		sys.stdout.close()
-		# exit:
-		if hasattr(os, '_exit') and os._exit:
-			os._exit(code)
-		else:
-			sys.exit(code)
+		try:
+			sys.stdout.flush()
+			# exit:
+			if hasattr(sys, 'exit') and sys.exit:
+				sys.exit(code)
+			else:
+				os._exit(code)
+		except (BrokenPipeError, IOError) as e: # pragma: no cover
+			if e.errno != 32: # closed / broken pipe
+				raise
 
 	@staticmethod
 	def exit(code=0):
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index f381576e..c45be849 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -224,9 +224,10 @@ def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover
 		sys.exit(0)
 
 try:
-    BrokenPipeError
+	BrokenPipeError = BrokenPipeError
 except NameError: # pragma: 3.x no cover
-    BrokenPipeError = IOError
+	BrokenPipeError = IOError
+
 __origLog = logging.Logger._log
 def __safeLog(self, level, msg, args, **kwargs):
 	"""Safe log inject to avoid possible errors by unsafe log-handlers, 

From e8ee3ba544c7eefd7b41e34ec3a44550c1280fbb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Sep 2020 11:36:54 +0200
Subject: [PATCH 12/16] resolves a bottleneck within transmitting of large data
 between server and client: speedup search of communications end-marker and
 increase max buffer size (up to 32KB)

---
 fail2ban/client/csocket.py       | 14 +++++++++-----
 fail2ban/tests/sockettestcase.py |  4 ++--
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py
index ab3e294b..88795674 100644
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@@ -48,7 +48,8 @@ class CSocket:
 	def send(self, msg, nonblocking=False, timeout=None):
 		# Convert every list member to string
 		obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL)
-		self.__csock.send(obj + CSPROTO.END)
+		self.__csock.send(obj)
+		self.__csock.send(CSPROTO.END)
 		return self.receive(self.__csock, nonblocking, timeout)
 
 	def settimeout(self, timeout):
@@ -81,9 +82,12 @@ class CSocket:
 		msg = CSPROTO.EMPTY
 		if nonblocking: sock.setblocking(0)
 		if timeout: sock.settimeout(timeout)
-		while msg.rfind(CSPROTO.END) == -1:
-			chunk = sock.recv(512)
-			if chunk in ('', b''): # python 3.x may return b'' instead of ''
-				raise RuntimeError("socket connection broken")
+		bufsize = 1024
+		while msg.rfind(CSPROTO.END, -32) == -1:
+			chunk = sock.recv(bufsize)
+			if not len(chunk):
+				raise socket.error(104, 'Connection reset by peer')
+			if chunk == CSPROTO.END: break
 			msg = msg + chunk
+			if bufsize < 32768: bufsize <<= 1
 		return loads(msg)
diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py
index 8cd22a41..2d414e5c 100644
--- a/fail2ban/tests/sockettestcase.py
+++ b/fail2ban/tests/sockettestcase.py
@@ -152,7 +152,7 @@ class Socket(LogCaptureTestCase):
 		org_handler = RequestHandler.found_terminator
 		try:
 			RequestHandler.found_terminator = lambda self: self.close()
-			self.assertRaisesRegexp(RuntimeError, r"socket connection broken", 
+			self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe",
 				lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10)))
 		finally:
 			RequestHandler.found_terminator = org_handler
@@ -168,7 +168,7 @@ class Socket(LogCaptureTestCase):
 		org_handler = RequestHandler.found_terminator
 		try:
 			RequestHandler.found_terminator = lambda self: TestMsgError()
-			#self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage)
+			#self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage)
 			self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error')
 		finally:
 			RequestHandler.found_terminator = org_handler

From f381b982465e3e9bf58f565cfea006fedc1989f1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Sep 2020 11:44:55 +0200
Subject: [PATCH 13/16] introduces new flavor `short` for `fail2ban-client
 status $jail short`: output total and current counts only, without banned IPs
 list in order to speedup it and to provide more clear output (gh-2819),
 flavor `basic` (still default) is unmodified for backwards compatibility; it
 can be changed later to `short`, so for full list of IPs in newer version one
 should better use:   - `fail2ban-client status $jail basic`   -
 `fail2ban-client get $jail banned` or `fail2ban-client banned`

---
 fail2ban/server/actions.py        | 14 ++++++++++----
 fail2ban/tests/actionstestcase.py |  2 ++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 3308d4b2..f14d8d7b 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -661,13 +661,19 @@ class Actions(JailThread, Mapping):
 		"""Status of current and total ban counts and current banned IP list.
 		"""
 		# TODO: Allow this list to be printed as 'status' output
-		supported_flavors = ["basic", "cymru"]
+		supported_flavors = ["short", "basic", "cymru"]
 		if flavor is None or flavor not in supported_flavors:
 			logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors))
 		# Always print this information (basic)
-		ret = [("Currently banned", self.__banManager.size()),
-			   ("Total banned", self.__banManager.getBanTotal()),
-			   ("Banned IP list", self.__banManager.getBanList())]
+		if flavor != "short":
+			banned = self.__banManager.getBanList()
+			cnt = len(banned)
+		else:
+			cnt = self.__banManager.size()
+		ret = [("Currently banned", cnt),
+			   ("Total banned", self.__banManager.getBanTotal())]
+		if flavor != "short":
+			ret += [("Banned IP list", banned)]
 		if flavor == "cymru":
 			cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 			ret += \
diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py
index d97d9921..532fe6ed 100644
--- a/fail2ban/tests/actionstestcase.py
+++ b/fail2ban/tests/actionstestcase.py
@@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase):
 		self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop')
 		self.assertEqual(self.__actions.status(),[("Currently banned", 0 ),
                ("Total banned", 0 ), ("Banned IP list", [] )])
+		self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ),
+               ("Total banned", 0 )])
 
 	def testAddActionPython(self):
 		self.__actions.add(

From d977d81ef75f96d41e8aa994aa8efd81f13bf795 Mon Sep 17 00:00:00 2001
From: "Sergey G. Brester" <serg.brester@sebres.de>
Date: Thu, 17 Sep 2020 12:39:08 +0200
Subject: [PATCH 14/16] action.d/abuseipdb.conf: removed broken link,
 simplified usage example, fixed typos

---
 config/action.d/abuseipdb.conf | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf
index d36cb3a9..ed958c86 100644
--- a/config/action.d/abuseipdb.conf
+++ b/config/action.d/abuseipdb.conf
@@ -21,14 +21,13 @@
 #
 # Example, for ssh bruteforce (in section [sshd] of `jail.local`): 
 #   action = %(known/action)s
-#            %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
+#            abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
 #
-# See below for catagories.
+# See below for categories.
 #
-# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)
 
-## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call.
+## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call.
 # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
 # ID	Title	Description
 # 3	  Fraud Orders
@@ -101,5 +100,5 @@ actionunban =
 # Notes    Your API key from abuseipdb.com
 # Values:  STRING  Default: None
 # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below.
-# You will need to set the catagory in the action call.
+# You will need to set the category in the action call.
 abuseipdb_apikey =

From f518d42c590f4ea3d8632937dc6ad7e875419b00 Mon Sep 17 00:00:00 2001
From: Nathan Henrie <nate@n8henrie.com>
Date: Wed, 12 Aug 2020 14:11:33 -0600
Subject: [PATCH 15/16] Add a note about `journalflags` options to
 `systemd-journal` backend

Also adds systemd backend configuration examples to jail.conf(5)

Closes #2696
---
 fail2ban/client/fail2banregex.py | 4 +++-
 man/jail.conf.5                  | 9 ++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index d0fe55dc..56da17dd 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -113,7 +113,9 @@ class _f2bOptParser(OptionParser):
 LOG:
     string                  a string representing a log line
     filename                path to a log file (/var/log/auth.log)
-    "systemd-journal"       search systemd journal (systemd-python required)
+    "systemd-journal"       search systemd journal. Optionally specify
+                            `systemd-journal[journalflags=X]` to determine
+                            which journals are used (systemd-python required)
 
 REGEX:
     string                  a string representing a 'failregex'
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 830c8aed..d7722124 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst
 uses a polling algorithm which does not require external libraries.
 .TP
 .B systemd
-uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config.
+uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples:
+.PP
+.RS
+.nf
+backend = systemd[journalpath=/run/log/journal/machine-1]
+backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"]
+backend = systemd[journalflags=1]
+.fi
 
 .SS Actions
 Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.:

From 24093de32daa05af5f178639095ac7106d55b544 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 23 Sep 2020 19:35:17 +0200
Subject: [PATCH 16/16] small amend (simplifying formatted help and man)

---
 fail2ban/client/fail2banregex.py | 22 +++++++++++-----------
 man/fail2ban-regex.1             |  9 +++++++--
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 56da17dd..e7a4e214 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -21,7 +21,6 @@ Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 
 This tools can test regular expressions for "fail2ban".
-
 """
 
 __author__ = "Fail2Ban Developers"
@@ -109,21 +108,22 @@ class _f2bOptParser(OptionParser):
 	def format_help(self, *args, **kwargs):
 		""" Overwritten format helper with full ussage."""
 		self.usage = ''
-		return "Usage: " + usage() + __doc__ + """
+		return "Usage: " + usage() + "\n" + __doc__ + """
 LOG:
-    string                  a string representing a log line
-    filename                path to a log file (/var/log/auth.log)
-    "systemd-journal"       search systemd journal. Optionally specify
-                            `systemd-journal[journalflags=X]` to determine
-                            which journals are used (systemd-python required)
+  string                a string representing a log line
+  filename              path to a log file (/var/log/auth.log)
+  systemd-journal       search systemd journal (systemd-python required),
+                        optionally with backend parameters, see `man jail.conf`
+                        for usage and examples (systemd-journal[journalflags=1]).
 
 REGEX:
-    string                  a string representing a 'failregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing a 'failregex'
+  filter                name of filter, optionally with options (sshd[mode=aggressive])
+  filename              path to a filter file (filter.d/sshd.conf)
 
 IGNOREREGEX:
-    string                  a string representing an 'ignoreregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  string                a string representing an 'ignoreregex'
+  filename              path to a filter file (filter.d/sshd.conf)
 \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n
 Report bugs to https://github.com/fail2ban/fail2ban/issues\n
 """ + __copyright__ + "\n"
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index bb89ef8c..3964126f 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -18,13 +18,18 @@ a string representing a log line
 filename
 path to a log file (\fI\,/var/log/auth.log\/\fP)
 .TP
-"systemd\-journal"
-search systemd journal (systemd\-python required)
+systemd\-journal
+search systemd journal (systemd\-python required),
+optionally with backend parameters, see `man jail.conf`
+for usage and examples (systemd\-journal[journalflags=1]).
 .SS "REGEX:"
 .TP
 string
 a string representing a 'failregex'
 .TP
+filter
+name of filter, optionally with options (sshd[mode=aggressive])
+.TP
 filename
 path to a filter file (filter.d/sshd.conf)
 .SS "IGNOREREGEX:"