From bdc2d07946d2b10ccf4634db6117e798ab079555 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 19:00:06 +0200
Subject: [PATCH 01/12] fix suhosin_log in common paths - log files should be
 separated using "\n": prevents to throw an error "File option must be 'head'
 or 'tail'", if jail suhosin will be enabled.

---
 config/paths-common.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index e2f08325..9072136c 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.
 
-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s
 
 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s

From a4b8f6e49e81340f3fdc31d2dd92935ea22160f5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 20:21:42 +0200
Subject: [PATCH 02/12] [part. cherry-picked from 0.10] invalid recursion check
 in substituteRecursiveTags: for example action `bsd-ipfw` produced
 ValueError('properties contain self referencing definitions and cannot be
 resolved...') test cases extended for exactly this case; closes gh-1417

---
 fail2ban/server/action.py        | 11 ++++++++---
 fail2ban/tests/actiontestcase.py |  7 +++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index de0c8efc..b1af659e 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -400,12 +400,16 @@ class CommandAction(ActionBase):
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = []
+				done = {}
+				last_found = tag
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or found_tag in done:
+					curdone = done.get(last_found)
+					if curdone is None:
+						done[last_found] = curdone = []
+					if found_tag == tag or found_tag in curdone:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						return False
@@ -417,7 +421,8 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag , tags[found_tag])
 					#logSys.log(5, 'value now: %s' % value)
-					done.append(found_tag)
+					curdone.append(found_tag)
+					last_found = found_tag
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 289d8896..e20a6341 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -29,6 +29,7 @@ import time
 import tempfile
 
 from ..server.action import CommandAction, CallingMap
+from ..server.actions import OrderedDict
 
 from .utils import LogCaptureTestCase
 from .utils import pid_exists
@@ -58,6 +59,12 @@ class CommandActionTest(LogCaptureTestCase):
 		# Unresolveable substition
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		# No-recursion, just multiple replacement of tag <T>, should be successful
+		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
+			self.assertEqual(CommandAction.substituteRecursiveTags(
+				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			)
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})

From cce63926ce9bb7b96fa1820e7893af67e0ac021a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 14:37:48 +0200
Subject: [PATCH 03/12] ChangeLog entry added

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 033cd9ec..9ac9a752 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
      - failregex of previous monit version merged as single expression.
    * filter.d/postfix.conf, filter.d/postfix-sasl.conf
      - extended failregex daemon part, matching also `postfix/smtps/smtpd` now (gh-1391)
+   * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
+     in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
+     Now tracks the actual list of the already substituted tags (per tag instead of single list)
 
 - New Features:
    * New Actions:

From 3e49522b7ab75833e412ec38d05e5edc9645993b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 20:07:19 +0200
Subject: [PATCH 04/12] fixes unexpected extra regex-space in generic
 `__prefix_line` (gh-1405, misleadingly committed in
 d2a953756802bd7cf63f5f5f792371f52f5cba8c); all optional spaces normalized in
 generic include `common.conf` + test cases are extended (using new example
 pseudo-filter and test log `zzz-generic-example`);

---
 ChangeLog                                     |  3 ++
 config/filter.d/common.conf                   |  8 +++---
 config/filter.d/zzz-generic-example.conf      | 17 +++++++++++
 fail2ban/tests/clientreadertestcase.py        |  7 +++--
 fail2ban/tests/files/logs/zzz-generic-example | 28 +++++++++++++++++++
 5 files changed, 56 insertions(+), 7 deletions(-)
 create mode 100644 config/filter.d/zzz-generic-example.conf
 create mode 100644 fail2ban/tests/files/logs/zzz-generic-example

diff --git a/ChangeLog b/ChangeLog
index 9ac9a752..44ba9d11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -18,6 +18,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * filter.d/common.conf
+     - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
+     - all optional spaces normalized in `common.conf`, test covered now
 
 - New Features:
    * New Actions:
diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 3e35f1d8..115fa96c 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -26,11 +26,11 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
 
 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
-__daemon_extra_re = (?:\[ID \d+ \S+\])
+__daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
+__daemon_combs_re = %(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -44,14 +44,14 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 
 # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
-__bsd_syslog_verbose = (<[^.]+\.[^.]+>)
+__bsd_syslog_verbose = <[^.]+\.[^.]+>
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s*
+__prefix_line = \s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
diff --git a/config/filter.d/zzz-generic-example.conf b/config/filter.d/zzz-generic-example.conf
new file mode 100644
index 00000000..421c117d
--- /dev/null
+++ b/config/filter.d/zzz-generic-example.conf
@@ -0,0 +1,17 @@
+# Fail2Ban generic example resp. test filter 
+#
+# Author: Serg G. Brester (sebres)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+[Definition]
+
+_daemon = test-demo
+
+failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
+ignoreregex = 
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 0a3734e5..b5c31c38 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -600,9 +600,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
-			filters = set(os.path.splitext(os.path.split(a)[1])[0]
-				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not a.endswith('common.conf'))
+			filters = (os.path.splitext(os.path.split(flt)[1])[0]
+				for flt in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+					if not flt.endswith('common.conf'))
+			filters = set(filter(lambda flt: not flt.startswith('zzz-'), filters))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
new file mode 100644
index 00000000..ecc5a1c6
--- /dev/null
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -0,0 +1,28 @@
+# -- _daemon with __pid_re, without __hostname --
+# failJSON: { "time": "2005-06-21T16:47:46", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:46 machine test-demo[13709]: F2B: failure from 192.0.2.1
+# -- _daemon with __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:48 test-demo[13709]: F2B: failure from 192.0.2.1
+
+# -- __kernel_prefix --
+# failJSON: { "time": "2005-06-21T16:47:50", "match": true , "host": "192.0.2.2" }
+Jun 21 16:47:50 machine kernel: [  970.699396] F2B: failure from 192.0.2.2
+
+# -- _daemon_re with and without __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:52", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:52 machine [test-demo] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:47:53", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:53 machine [test-demo][13709] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:00", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:00 machine test-demo(pam_unix) F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:02 machine test-demo(pam_unix)[13709] F2B: failure from 192.0.2.3
+
+
+# -- all common definitions together (bsdverbose hostname kernel_prefix vserver tag daemon_id space) --
+# failJSON: { "time": "2005-06-21T16:55:01", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
+# -- the same as above with additional spaces around --
+# failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3

From de813acf5104dc00c0086b544c6fe82f428d4ca8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 11:33:49 +0200
Subject: [PATCH 05/12] extends generic `__prefix_line` with optional brackets
 for the date ambit (gh-1421), added new parameter `__date_ambit` + test case
 added;

---
 ChangeLog                                     | 2 ++
 config/filter.d/common.conf                   | 4 +++-
 fail2ban/tests/files/logs/zzz-generic-example | 3 +++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 44ba9d11..d52e1cff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -21,6 +21,8 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * filter.d/common.conf
      - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
      - all optional spaces normalized in `common.conf`, test covered now
+     - generic `__prefix_line` extended with optional brackets for the date ambit (gh-1421),
+       added new parameter `__date_ambit`
 
 - New Features:
    * New Actions:
diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 115fa96c..23fd1d5a 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -46,12 +46,14 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
 __bsd_syslog_verbose = <[^.]+\.[^.]+>
 
+__date_ambit = \[\]
+
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
+__prefix_line = (?:%(__date_ambit)s)?\s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index ecc5a1c6..e3480b63 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -26,3 +26,6 @@ Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-de
 # -- the same as above with additional spaces around --
 # failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
 Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3
+# -- the same as above with brackets as date ambit --
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
+[Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3

From cb4f9be8b2748e940303cff77d72ab7fdcf06a53 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 11:54:08 +0200
Subject: [PATCH 06/12] the date brackets removed from filters using
 `__prefix_line`, because `__prefix_line` already contains the date ambit;

---
 config/filter.d/asterisk.conf | 20 ++++++++++----------
 config/filter.d/nsd.conf      |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 3975fb29..01063efa 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -16,17 +16,17 @@ __pid_re = (?:\[\d+\])
 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 
 # All Asterisk log messages begin like this:
-log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)?
+log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*( in \w+:)?
 
-failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
-            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )Ext\. s: "Rejecting unknown SIP connection from <HOST>"$
+failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+            ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
+            ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
+            ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
+            ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
 
 ignoreregex =
 
diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf
index 70b41ca4..8f32f7be 100644
--- a/config/filter.d/nsd.conf
+++ b/config/filter.d/nsd.conf
@@ -22,7 +22,7 @@ _daemon = nsd
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 
-failregex =  ^\[\]%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
-              ^\[\]%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
+failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
+             ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =

From 25af11215b71dec2197b8df9ee1aa9f813965aca Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 20:08:46 +0200
Subject: [PATCH 07/12] test case for generic common moved to
 `./fail2ban/tests/config/filter.d/zzz-generic-example.conf` to prevent
 shipping it with fail2ban installations

---
 fail2ban/tests/clientreadertestcase.py        |  7 +++---
 .../config}/filter.d/zzz-generic-example.conf |  2 +-
 fail2ban/tests/samplestestcase.py             | 25 +++++++++++--------
 3 files changed, 19 insertions(+), 15 deletions(-)
 rename {config => fail2ban/tests/config}/filter.d/zzz-generic-example.conf (85%)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index b5c31c38..0a3734e5 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -600,10 +600,9 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
-			filters = (os.path.splitext(os.path.split(flt)[1])[0]
-				for flt in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not flt.endswith('common.conf'))
-			filters = set(filter(lambda flt: not flt.startswith('zzz-'), filters))
+			filters = set(os.path.splitext(os.path.split(a)[1])[0]
+				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+					if not a.endswith('common.conf'))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
diff --git a/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
similarity index 85%
rename from config/filter.d/zzz-generic-example.conf
rename to fail2ban/tests/config/filter.d/zzz-generic-example.conf
index 421c117d..a59ccb1e 100644
--- a/config/filter.d/zzz-generic-example.conf
+++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
@@ -7,7 +7,7 @@
 
 # Read common prefixes. If any customizations available -- read them from
 # common.local
-before = common.conf
+before = ../../../../config/filter.d/common.conf
 
 [Definition]
 
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 9e6c0ee7..a40a11da 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -35,6 +35,7 @@ from ..server.filter import Filter
 from ..client.filterreader import FilterReader
 from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR
 
+TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 
@@ -60,11 +61,11 @@ class FilterSamplesRegex(unittest.TestCase):
 			"Expected more FilterSampleRegexs tests")
 
 
-def testSampleRegexsFactory(name):
+def testSampleRegexsFactory(name, basedir):
 	def testFilter(self):
 
 		# Check filter exists
-		filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR)
+		filterConf = FilterReader(name, "jail", {}, basedir=basedir)
 		self.assertEqual(filterConf.getFile(), name)
 		self.assertEqual(filterConf.getJailName(), "jail")
 		filterConf.read()
@@ -147,11 +148,15 @@ def testSampleRegexsFactory(name):
 
 	return testFilter
 
-for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'),
-					  os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
-	filterName = filter_.rpartition(".")[0]
-	if not filterName.startswith('.'):
-		setattr(
-			FilterSamplesRegex,
-			"testSampleRegexs%s" % filterName.upper(),
-			testSampleRegexsFactory(filterName))
+for basedir_, filter_ in (
+	(CONFIG_DIR, lambda x: not x.endswith('common.conf') and x.endswith('.conf')),
+	(TEST_CONFIG_DIR, lambda x: x.startswith('zzz-') and x.endswith('.conf')),
+):
+	for filter_ in filter(filter_,
+						  os.listdir(os.path.join(basedir_, "filter.d"))):
+		filterName = filter_.rpartition(".")[0]
+		if not filterName.startswith('.'):
+			setattr(
+				FilterSamplesRegex,
+				"testSampleRegexs%s" % filterName.upper(),
+				testSampleRegexsFactory(filterName, basedir_))

From 52377984cd415c089438bdfbcc2c86e344f8ca59 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 17:01:44 +0200
Subject: [PATCH 08/12] back to mandatory space, ungrouping of sub parameters
 in `__prefix_line` + small code review;

---
 config/filter.d/common.conf | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 23fd1d5a..586f428a 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -30,7 +30,7 @@ __daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = %(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -46,14 +46,16 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
 __bsd_syslog_verbose = <[^.]+\.[^.]+>
 
-__date_ambit = \[\]
+__vserver = @vserver_\S+
+
+__date_ambit = (?:\[\])
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = (?:%(__date_ambit)s)?\s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
+__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap

From 932708de9e6a3ba5089116cc3c1ee456ba3b6855 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 19:03:32 +0200
Subject: [PATCH 09/12] fixed --pidfile bug, introduced in gh-1322:
 gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon,
 not argument of fail2ban (see gh-1434) closes gh-1434

---
 ChangeLog          | 2 ++
 files/gentoo-initd | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9ac9a752..55fb8aff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -18,6 +18,8 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, 
+     not argument of fail2ban (see gh-1434)
 
 - New Features:
    * New Actions:
diff --git a/files/gentoo-initd b/files/gentoo-initd
index b7e1067b..c5b5b702 100755
--- a/files/gentoo-initd
+++ b/files/gentoo-initd
@@ -34,15 +34,15 @@ start() {
 	# remove stalled sock file after system crash
 	# bug 347477
 	rm -f /var/run/fail2ban/fail2ban.sock || return 1
-	start-stop-daemon --start --exec ${FAIL2BAN} start \
-	--pidfile /var/run/fail2ban/fail2ban.pid
+	start-stop-daemon --start --pidfile /var/run/fail2ban/fail2ban.pid \
+	-- ${FAIL2BAN} start
 	eend $? "Failed to start fail2ban"
 }
 
 stop() {
 	ebegin "Stopping fail2ban"
-	start-stop-daemon --stop --exec ${FAIL2BAN} stop \
-	--pidfile /var/run/fail2ban/fail2ban.pid
+	start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid \
+	-- ${FAIL2BAN} stop
 	eend $? "Failed to stop fail2ban"
 }
 

From db9e7240386b7fdee5d796138f7118d01e3168e0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 12:03:35 +0200
Subject: [PATCH 10/12] extremely speedup of all database operations: -
 (synchronous = OFF) write data through OS without syncing - (journal_mode =
 MEMORY) use memory for the transaction logging

---
 fail2ban/server/database.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 7de87554..c0e9bf46 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -182,8 +182,11 @@ class Fail2BanDb(object):
 			raise
 
 		cur = self._db.cursor()
-		cur.execute("PRAGMA foreign_keys = ON;")
-
+		cur.execute("PRAGMA foreign_keys = ON")
+		# speedup: write data through OS without syncing (no wait):
+		cur.execute("PRAGMA synchronous = OFF")
+		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
+		cur.execute("PRAGMA journal_mode = MEMORY")
 		try:
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:

From baafac36a4c578769d53240147d2777b6414ad1c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 12:42:18 +0200
Subject: [PATCH 11/12] ChangeLog entry

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 433026f3..76719f16 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * New Actions:
      - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
 - Enhancements:
+   * Extreme speedup of all sqlite database operations (gh-1436),
+     by using of following sqlite options:
+     - (synchronous = OFF) write data through OS without syncing
+     - (journal_mode = MEMORY) use memory for the transaction logging
+     - (temp_store = MEMORY) temporary tables and indices are kept in memory
    * journald journalmatch for pure-ftpd (gh-1362)
    * Add additional regex filter for dovecot ldap authentication failures (gh-1370)
    * added additional regex filters for exim (gh-1371)

From 1718c8dbe92030617042db2c7a01e996cf16cab7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 13:26:51 +0200
Subject: [PATCH 12/12] pypy: switch journal mode after upgrade (save it during
 the upgrade), to prevent errors like "database table is locked"

---
 fail2ban/server/database.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index c0e9bf46..560fbfe5 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -181,12 +181,24 @@ class Fail2BanDb(object):
 				filename, e.args[0])
 			raise
 
+		# differentiate pypy: switch journal mode later (save it during the upgrade), 
+		# to prevent errors like "database table is locked":
+		try:
+			import __pypy__
+			pypy = True
+		except ImportError:
+			pypy = False
+
 		cur = self._db.cursor()
 		cur.execute("PRAGMA foreign_keys = ON")
 		# speedup: write data through OS without syncing (no wait):
 		cur.execute("PRAGMA synchronous = OFF")
 		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
-		cur.execute("PRAGMA journal_mode = MEMORY")
+		if not pypy:
+			cur.execute("PRAGMA journal_mode = MEMORY")
+		# speedup: temporary tables and indices are kept in memory:
+		cur.execute("PRAGMA temp_store = MEMORY")
+
 		try:
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:
@@ -205,6 +217,9 @@ class Fail2BanDb(object):
 						Fail2BanDb.__version__, version, newversion)
 					raise RuntimeError('Failed to fully update')
 		finally:
+			# pypy: set journal mode after possible upgrade db:
+			if pypy:
+				cur.execute("PRAGMA journal_mode = MEMORY")
 			cur.close()
 
 	@property
@@ -247,13 +262,14 @@ class Fail2BanDb(object):
 
 		A timestamped backup is also created prior to attempting the update.
 		"""
-		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
-		shutil.copyfile(self.filename, self._dbBackupFilename)
-		logSys.info("Database backup created: %s", self._dbBackupFilename)
 		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
 						"Attempt to travel to future version of database ...how did you get here??")
 
+		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+		shutil.copyfile(self.filename, self._dbBackupFilename)
+		logSys.info("Database backup created: %s", self._dbBackupFilename)
+
 		if version < 2:
 			cur.executescript("BEGIN TRANSACTION;"
 						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"