From bdc2d07946d2b10ccf4634db6117e798ab079555 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 19:00:06 +0200
Subject: [PATCH 01/61] fix suhosin_log in common paths - log files should be
 separated using "\n": prevents to throw an error "File option must be 'head'
 or 'tail'", if jail suhosin will be enabled.

---
 config/paths-common.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index e2f08325..9072136c 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.
 
-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s
 
 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s

From 0c2eeee8c7fc4b3a26083ef2f04ebde0c96560c6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 12:53:42 +0200
Subject: [PATCH 02/61] BF: fail2ban-client can't unserialize IPAddr objects -
 added IPAddr pickle-handler, that simple wrap IPAddr to the str

---
 fail2ban/server/ipdns.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index bca03428..880e3f30 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -233,6 +233,14 @@ class IPAddr(object):
 
 	def __str__(self):
 		return self.ntoa
+
+	def __reduce__(self):
+		"""IPAddr pickle-handler, that simple wrap IPAddr to the str
+
+		Returns a string as instance to be pickled, because fail2ban-client can't
+		unserialize IPAddr objects
+		"""
+		return (str, (self.ntoa,))
 	
 	@property
 	def addr(self):

From 4b5b16cd9fc5baa8b43415de072d7ba74377943e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 15:33:15 +0200
Subject: [PATCH 03/61] allow using of IPv6 address style mask (analog to the
 IPv4), for example: `2606:28ff::/ffff:ff80::` -> `2606:2880::/25` fast
 calculating of maskplen using map table MAP_ADDR2MASKPLEN, with
 pre-calculated addr->maskplen values; test cases extended;

---
 fail2ban/server/ipdns.py         | 28 ++++++++++++++++++++--------
 fail2ban/tests/filtertestcase.py | 13 +++++++++++++
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 880e3f30..8109c21a 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -171,7 +171,7 @@ class IPAddr(object):
 		# IP address without CIDR mask
 		if len(s) > 2:
 			raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,))
-		if "." in s[1]: # 255.255.255.0 style mask
+		if "." in s[1] or ":" in s[1]: # 255.255.255.0 resp. ffff:: style mask
 			s[1] = IPAddr.masktoplen(s[1])
 		s[1] = long(s[1])
 		return s
@@ -235,7 +235,7 @@ class IPAddr(object):
 		return self.ntoa
 
 	def __reduce__(self):
-		"""IPAddr pickle-handler, that simple wrap IPAddr to the str
+		"""IPAddr pickle-handler, that simply wraps IPAddr to the str
 
 		Returns a string as instance to be pickled, because fail2ban-client can't
 		unserialize IPAddr objects
@@ -392,17 +392,29 @@ class IPAddr(object):
 		
 		return (self.addr & mask) == net.addr
 
+	# Pre-calculated map: addr to maskplen
+	def __getMaskMap():
+		m6 = (1 << 128)-1
+		m4 = (1 << 32)-1
+		mmap = {m6: 128, m4: 32, 0: 0}
+		m = 0
+		for i in xrange(0, 128):
+			m |= 1 << i
+			if i < 32:
+				mmap[m ^ m4] = 32-1-i
+			mmap[m ^ m6] = 128-1-i
+		return mmap
+
+	MAP_ADDR2MASKPLEN = __getMaskMap()
+
 	@property
 	def maskplen(self):
 		mplen = 0
 		if self._maskplen is not None:
 			return self._maskplen
-		maddr = self._addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			mplen += 1
+		mplen = IPAddr.MAP_ADDR2MASKPLEN.get(self._addr)
+		if mplen is None:
+			raise ValueError("invalid mask %r, no plen representation" % (str(self),))
 		self._maskplen = mplen
 		return mplen
 		
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 5e493f83..1d4db3a2 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1452,11 +1452,24 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(IPAddr('93.184.0.1', 24).ntoa, '93.184.0.0/24')
 		self.assertEqual(IPAddr('192.168.1.0/255.255.255.128').ntoa, '192.168.1.0/25')
 
+		self.assertEqual(IPAddr('93.184.0.1/32').ntoa, '93.184.0.1')
+		self.assertEqual(IPAddr('93.184.0.1/255.255.255.255').ntoa, '93.184.0.1')
+
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8::', 120)), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8::', 120).ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::', 25)), '2606:2880::/25')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ff80::')), '2606:2880::/25')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff::')), 
+			'2606:28ff:220:1:248:1893:25c8:0/112')
+
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/128')), 
+			'2606:28ff:220:1:248:1893:25c8:0')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')), 
+			'2606:28ff:220:1:248:1893:25c8:0')
+
 	def testIPAddr_CIDR_Repr(self):
 		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
 			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]

From ec6032d934f72fc2a373a630e44499505b561519 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 17:16:09 +0200
Subject: [PATCH 04/61] prevent to fail stock configs test case, if any jail
 custom config does not have own test log-file (perhaps not clean copy)

---
 fail2ban/tests/servertestcase.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 90f7aaf2..b79a5652 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1021,7 +1021,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						cmd[2] = 'polling'
 					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
 					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
-						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
+						fn = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
+						# fallback to testcase01 if jail has not an own test log-file (currently should be no matter):
+						if not os.path.exists(fn): # pragma: no cover
+							fn = os.path.join(TEST_FILES_DIR, 'testcase01.log')
+						cmd[3] = fn
 					# if fast add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'

From 14c31d8c58ac1b5beb3974a8b3ecd82ab19d7a08 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 12 May 2016 18:52:20 -0400
Subject: [PATCH 05/61] DOC: minor PEP8ing and comments enhancements

---
 fail2ban/tests/clientreadertestcase.py |  2 +-
 fail2ban/tests/servertestcase.py       | 20 ++++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 0edbc69e..5e66e25e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -45,7 +45,7 @@ TEST_FILES_DIR_SHARE_CFG = {}
 from .utils import CONFIG_DIR
 CONFIG_DIR_SHARE_CFG = unittest.F2B.share_config
 
-STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
+STOCK = os.path.exists(os.path.join('config', 'fail2ban.conf'))
 
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
 IMPERFECT_CONFIG_SHARE_CFG = {}
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index b79a5652..214520a6 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -844,7 +844,7 @@ class TransmitterLogging(TransmitterBase):
 				   outCode=1,
 				   outValue=Exception('Failed to change log target'),
 				   repr_=True # Exceptions are not comparable apparently
-                                  )
+                                )
 			  }[platform.system() in ('Linux',) and os.path.exists('/dev/log')]
 		)
 
@@ -1001,7 +1001,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 	if STOCK:
 
 		def testCheckStockJailActions(self):
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			# we are running tests from root project dir atm
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg)
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
 			stream = jails.convert(allow_no_files=True)
@@ -1019,14 +1020,17 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# change to the fast init backend:
 					if cmd[0] == 'add':
 						cmd[2] = 'polling'
-					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
+					# change log path to test log of the jail
+					# (to prevent "Permission denied" on /var/logs/ for test-user):
 					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
 						fn = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
-						# fallback to testcase01 if jail has not an own test log-file (currently should be no matter):
-						if not os.path.exists(fn): # pragma: no cover
+						# fallback to testcase01 if jail has no its own test log-file
+						# (should not matter really):
+						if not os.path.exists(fn):  # pragma: no cover
 							fn = os.path.join(TEST_FILES_DIR, 'testcase01.log')
 						cmd[3] = fn
-					# if fast add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					# if fast add dummy regex to prevent too long compile of all regexp
+					# (we don't use it in this test at all):
 					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
 					):
@@ -1035,7 +1039,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
 					try:
 						cmdHandler(cmd)
-					except Exception, e: # pragma: no cover
+					except Exception, e:  # pragma: no cover
 						self.fail("Command %r has failed. Received %r" % (cmd, e))
 
 			# jails = server._Server__jails
@@ -1318,7 +1322,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
- 						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',

From 3d3735706b71da567de00b0158beba8915da4c94 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 20:21:42 +0200
Subject: [PATCH 06/61] invalid recursion check in substituteRecursiveTags: for
 example action `bsd-ipfw` produced ValueError('properties contain self
 referencing definitions and cannot be resolved...') test cases extended for
 exactly this case and for all stock actions; closes gh-1417

---
 fail2ban/server/action.py        | 14 +++++---
 fail2ban/tests/actiontestcase.py |  7 ++++
 fail2ban/tests/servertestcase.py | 57 +++++++++++++++++++++++++++++++-
 3 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index cdd4e28c..ba3d4633 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -366,17 +366,22 @@ class CommandAction(ActionBase):
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = []
+				done = {}
+				last_found = tag
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or found_tag in done:
+					curdone = done.get(last_found)
+					if curdone is None:
+						done[last_found] = curdone = []
+					if found_tag == tag or found_tag in curdone:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						raise ValueError(
 							"properties contain self referencing definitions "
-							"and cannot be resolved, fail tag: %s value: %s" % (tag, value))
+							"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
+							(tag, found_tag, curdone, value))
 					repl = None
 					if found_tag not in cls._escapedTags:
 						repl = tags.get(found_tag + '?' + conditional)
@@ -390,7 +395,8 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
-					done.append(found_tag)
+					curdone.append(found_tag)
+					last_found = found_tag
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index f3778022..00a4d6be 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -30,6 +30,7 @@ import time
 import unittest
 
 from ..server.action import CommandAction, CallingMap
+from ..server.actions import OrderedDict
 from ..server.utils import Utils
 
 from .utils import LogCaptureTestCase
@@ -65,6 +66,12 @@ class CommandActionTest(LogCaptureTestCase):
 			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertRaises(ValueError,
 			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		# No-recursion, just multiple replacement of tag <T>, should be successful
+		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
+			self.assertEqual(CommandAction.substituteRecursiveTags(
+				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			)
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 214520a6..cc14e97f 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -998,6 +998,38 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
 		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
 
+	def _testExecActions(self, server):
+		jails = server._Server__jails
+		for jail in jails:
+			# print(jail, jails[jail])
+			for a in jails[jail].actions:
+				action = jails[jail].actions[a]
+				logSys.debug('# ' + ('=' * 50))
+				logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+				logSys.debug('# ' + ('=' * 50))
+				# we can currently test only command actions:
+				if not isinstance(action, _actions.CommandAction): continue
+				# wrap default command processor, just log if (heavy)debug:
+				action.executeCmd = self._executeCmd
+				# test start :
+				logSys.debug('# === start ==='); self.pruneLog()
+				action.start()
+				# test ban ip4 :
+				logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+				action.ban({'ip': IPAddr('192.0.2.1')})
+				# test unban ip4 :
+				logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+				action.unban({'ip': IPAddr('192.0.2.1')})
+				# test ban ip6 :
+				logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+				action.ban({'ip': IPAddr('2001:DB8::')})
+				# test unban ip6 :
+				logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+				action.unban({'ip': IPAddr('2001:DB8::')})
+				# test stop :
+				logSys.debug('# === stop ==='); self.pruneLog()
+				action.stop()
+
 	if STOCK:
 
 		def testCheckStockJailActions(self):
@@ -1046,6 +1078,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			# for j in jails:
 			# 	print(j, jails[j])
 
+			# test default stock actions sepecified in all stock jails:
+			if not unittest.F2B.fast:
+				self._testExecActions(server)
+
 		def getDefaultJailStream(self, jail, act):
 			act = act.replace('%(__name__)s', jail)
 			actName, actOpt = JailReader.extractOptions(act)
@@ -1061,6 +1097,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			stream.extend(action.convert())
 			return stream
 
+		def testCheckStockAllActions(self):
+			unittest.F2B.SkipIfFast()
+			import glob
+
+			server = TestServer()
+			transm = server._Server__transm
+
+			for actCfg in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')):
+				act = os.path.basename(actCfg).replace('.conf', '')
+				# transmit artifical jail with each action to the server:
+				stream = self.getDefaultJailStream('j-'+act, act)
+				for cmd in stream:
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
+				# test executing action commands:
+				self._testExecActions(server)
+
+
 		def testCheckStockCommandActions(self):
 			# test cases to check valid ipv4/ipv6 action definition, tuple with (('jail', 'action[params]', 'tests', ...)
 			# where tests is a dictionary contains:
@@ -1347,7 +1402,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# for cmd in stream:
 				# 	print(cmd)
 
-				# filter all start commands (we want not start all jails):
+				# transmit jail to the server:
 				for cmd in stream:
 					# command to server:
 					ret, res = transm.proceed(cmd)

From a4b8f6e49e81340f3fdc31d2dd92935ea22160f5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 20:21:42 +0200
Subject: [PATCH 07/61] [part. cherry-picked from 0.10] invalid recursion check
 in substituteRecursiveTags: for example action `bsd-ipfw` produced
 ValueError('properties contain self referencing definitions and cannot be
 resolved...') test cases extended for exactly this case; closes gh-1417

---
 fail2ban/server/action.py        | 11 ++++++++---
 fail2ban/tests/actiontestcase.py |  7 +++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index de0c8efc..b1af659e 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -400,12 +400,16 @@ class CommandAction(ActionBase):
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = []
+				done = {}
+				last_found = tag
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or found_tag in done:
+					curdone = done.get(last_found)
+					if curdone is None:
+						done[last_found] = curdone = []
+					if found_tag == tag or found_tag in curdone:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						return False
@@ -417,7 +421,8 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag , tags[found_tag])
 					#logSys.log(5, 'value now: %s' % value)
-					done.append(found_tag)
+					curdone.append(found_tag)
+					last_found = found_tag
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 289d8896..e20a6341 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -29,6 +29,7 @@ import time
 import tempfile
 
 from ..server.action import CommandAction, CallingMap
+from ..server.actions import OrderedDict
 
 from .utils import LogCaptureTestCase
 from .utils import pid_exists
@@ -58,6 +59,12 @@ class CommandActionTest(LogCaptureTestCase):
 		# Unresolveable substition
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		# No-recursion, just multiple replacement of tag <T>, should be successful
+		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
+			self.assertEqual(CommandAction.substituteRecursiveTags(
+				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			)
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})

From cce63926ce9bb7b96fa1820e7893af67e0ac021a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 14:37:48 +0200
Subject: [PATCH 08/61] ChangeLog entry added

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 033cd9ec..9ac9a752 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
      - failregex of previous monit version merged as single expression.
    * filter.d/postfix.conf, filter.d/postfix-sasl.conf
      - extended failregex daemon part, matching also `postfix/smtps/smtpd` now (gh-1391)
+   * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
+     in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
+     Now tracks the actual list of the already substituted tags (per tag instead of single list)
 
 - New Features:
    * New Actions:

From 3e49522b7ab75833e412ec38d05e5edc9645993b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 20:07:19 +0200
Subject: [PATCH 09/61] fixes unexpected extra regex-space in generic
 `__prefix_line` (gh-1405, misleadingly committed in
 d2a953756802bd7cf63f5f5f792371f52f5cba8c); all optional spaces normalized in
 generic include `common.conf` + test cases are extended (using new example
 pseudo-filter and test log `zzz-generic-example`);

---
 ChangeLog                                     |  3 ++
 config/filter.d/common.conf                   |  8 +++---
 config/filter.d/zzz-generic-example.conf      | 17 +++++++++++
 fail2ban/tests/clientreadertestcase.py        |  7 +++--
 fail2ban/tests/files/logs/zzz-generic-example | 28 +++++++++++++++++++
 5 files changed, 56 insertions(+), 7 deletions(-)
 create mode 100644 config/filter.d/zzz-generic-example.conf
 create mode 100644 fail2ban/tests/files/logs/zzz-generic-example

diff --git a/ChangeLog b/ChangeLog
index 9ac9a752..44ba9d11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -18,6 +18,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * filter.d/common.conf
+     - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
+     - all optional spaces normalized in `common.conf`, test covered now
 
 - New Features:
    * New Actions:
diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 3e35f1d8..115fa96c 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -26,11 +26,11 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
 
 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
-__daemon_extra_re = (?:\[ID \d+ \S+\])
+__daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
+__daemon_combs_re = %(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -44,14 +44,14 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 
 # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
-__bsd_syslog_verbose = (<[^.]+\.[^.]+>)
+__bsd_syslog_verbose = <[^.]+\.[^.]+>
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s*
+__prefix_line = \s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
diff --git a/config/filter.d/zzz-generic-example.conf b/config/filter.d/zzz-generic-example.conf
new file mode 100644
index 00000000..421c117d
--- /dev/null
+++ b/config/filter.d/zzz-generic-example.conf
@@ -0,0 +1,17 @@
+# Fail2Ban generic example resp. test filter 
+#
+# Author: Serg G. Brester (sebres)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+[Definition]
+
+_daemon = test-demo
+
+failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
+ignoreregex = 
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 0a3734e5..b5c31c38 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -600,9 +600,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
-			filters = set(os.path.splitext(os.path.split(a)[1])[0]
-				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not a.endswith('common.conf'))
+			filters = (os.path.splitext(os.path.split(flt)[1])[0]
+				for flt in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+					if not flt.endswith('common.conf'))
+			filters = set(filter(lambda flt: not flt.startswith('zzz-'), filters))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
new file mode 100644
index 00000000..ecc5a1c6
--- /dev/null
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -0,0 +1,28 @@
+# -- _daemon with __pid_re, without __hostname --
+# failJSON: { "time": "2005-06-21T16:47:46", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:46 machine test-demo[13709]: F2B: failure from 192.0.2.1
+# -- _daemon with __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:48 test-demo[13709]: F2B: failure from 192.0.2.1
+
+# -- __kernel_prefix --
+# failJSON: { "time": "2005-06-21T16:47:50", "match": true , "host": "192.0.2.2" }
+Jun 21 16:47:50 machine kernel: [  970.699396] F2B: failure from 192.0.2.2
+
+# -- _daemon_re with and without __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:52", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:52 machine [test-demo] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:47:53", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:53 machine [test-demo][13709] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:00", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:00 machine test-demo(pam_unix) F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:02 machine test-demo(pam_unix)[13709] F2B: failure from 192.0.2.3
+
+
+# -- all common definitions together (bsdverbose hostname kernel_prefix vserver tag daemon_id space) --
+# failJSON: { "time": "2005-06-21T16:55:01", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
+# -- the same as above with additional spaces around --
+# failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3

From e0924e0d1bb948141cb4efd494d188436355032e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 21:44:07 +0200
Subject: [PATCH 10/61] test case fix (always sort result of
 `DNSUtils.textToIp`, because order of result from `socket.getaddrinfo` is
 undefined (system depended)

---
 fail2ban/tests/filtertestcase.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 1d4db3a2..4f5dbb55 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1357,9 +1357,11 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
-		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		# sort ipaddr, IPv4 is always smaller as IPv6
+		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
-		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		# sort ipaddr, IPv4 is always smaller as IPv6
+		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1371,7 +1373,8 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		for s in hostnames:
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
-				self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+				# sort ipaddr, IPv4 is always smaller as IPv6
+				self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 

From cb280b817f89386546c58109f2138fea6e0cfc08 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 22:16:35 +0200
Subject: [PATCH 11/61] csocket multi-set fix: prevent to convert `list`,
 `dict`, `set` during transfer (send), this offers a sending of 'multi-set'
 arrays (missed by cherry-picking from multi-set branch)

---
 fail2ban/client/csocket.py     | 4 +++-
 fail2ban/server/server.py      | 4 ++++
 fail2ban/server/transmitter.py | 3 ++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py
index 2e22e5ee..45d58cc6 100644
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@@ -44,7 +44,9 @@ class CSocket:
 	
 	def send(self, msg):
 		# Convert every list member to string
-		obj = dumps([str(m) for m in msg], HIGHEST_PROTOCOL)
+		obj = dumps(map(
+			lambda m: str(m) if not isinstance(m, (list, dict, set)) else m, msg),
+		  HIGHEST_PROTOCOL)
 		self.__csock.send(obj + CSPROTO.END)
 		return self.receive(self.__csock)
 
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 7f75c347..def796a5 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -266,8 +266,10 @@ class Server:
 		flt = self.__jails[name].filter
 		if multiple:
 			for value in value:
+				logSys.debug("  failregex: %r", value)
 				flt.addFailRegex(value)
 		else:
+			logSys.debug("  failregex: %r", value)
 			flt.addFailRegex(value)
 	
 	def delFailRegex(self, name, index):
@@ -280,8 +282,10 @@ class Server:
 		flt = self.__jails[name].filter
 		if multiple:
 			for value in value:
+				logSys.debug("  ignoreregex: %r", value)
 				flt.addIgnoreRegex(value)
 		else:
+			logSys.debug("  ignoreregex: %r", value)
 			flt.addIgnoreRegex(value)
 	
 	def delIgnoreRegex(self, name, index):
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 29d6d189..af2d1b53 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -52,7 +52,7 @@ class Transmitter:
 	
 	def proceed(self, command):
 		# Deserialize object
-		logSys.debug("Command: " + repr(command))
+		logSys.debug("Command: %r", command)
 		try:
 			ret = self.__commandHandler(command)
 			ack = 0, ret
@@ -263,6 +263,7 @@ class Transmitter:
 			action = self.__server.getAction(name, actionname)
 			if multiple:
 				for cmd in command[3]:
+					logSys.debug("  %r", cmd)
 					actionkey = cmd[0]
 					if callable(getattr(action, actionkey, None)):
 						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}

From d4201480552da3e69a6586c271f097ea18b98f41 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 22:53:57 +0200
Subject: [PATCH 12/61] database: always explicit convert `ip` to `str`,
 because may be an IPAddr, that will be unsupported type by bind parameter (as
 long as we've found any default wrapper handler for sqlite3)

---
 fail2ban/server/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 6a3d87c3..a6b3feb1 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -455,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip)
+			queryArgs.append(str(ip))
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)

From 2b6f8737a72703426149dfd42bac4f74b1e8a6ce Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 13 May 2016 17:17:46 -0400
Subject: [PATCH 13/61] ENH: version Fail2Ban in this branch as 0.10.0 alpha 1

---
 fail2ban/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/version.py b/fail2ban/version.py
index 140ca959..db4daa97 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.9.4.dev0"
+version = "0.10.0a1"

From 7e54cee8d61be84bfb8f3eeff70f9f86b90eef9e Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 21:36:27 -0400
Subject: [PATCH 14/61] updated firewallcmd actions

---
 config/action.d/firewallcmd-allports.conf     | 28 +++-----
 config/action.d/firewallcmd-common.conf       | 66 +++++++++++++++++++
 config/action.d/firewallcmd-ipset.conf        |  6 +-
 config/action.d/firewallcmd-multiport.conf    | 51 +++-----------
 config/action.d/firewallcmd-new.conf          | 29 +++-----
 config/action.d/firewallcmd-rich-logging.conf | 34 +++-------
 config/action.d/firewallcmd-rich-rules.conf   | 35 +++-------
 7 files changed, 119 insertions(+), 130 deletions(-)
 create mode 100644 config/action.d/firewallcmd-common.conf

diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf
index 571d5ba6..de0e7f91 100644
--- a/config/action.d/firewallcmd-allports.conf
+++ b/config/action.d/firewallcmd-allports.conf
@@ -6,34 +6,26 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-recidive$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Default name of the chain
-#
-name = default
-
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #
diff --git a/config/action.d/firewallcmd-common.conf b/config/action.d/firewallcmd-common.conf
new file mode 100644
index 00000000..106ef216
--- /dev/null
+++ b/config/action.d/firewallcmd-common.conf
@@ -0,0 +1,66 @@
+# Fail2Ban configuration file
+#
+# Author: Donald Yandt
+#
+
+[Init]
+
+# Option:  name
+# Notes    Default name of the chain
+# Values:  STRING
+name = default
+
+# Option:  family(ipv4)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv4
+
+# Option:  chain
+# Notes    specifies the firewalld chain to which the Fail2Ban rules should be
+#          added
+# Values:  STRING  Default: INPUT_direct
+chain = INPUT_direct
+
+# Option:  zone
+# Notes    use command firewall-cmd --get-active-zones to see a list of all active zones. See firewalld man pages for more information on zones
+# Values:  STRING  Default: public
+zone = public
+
+# Option:  service
+# Notes    use command firewall-cmd --get-services to see a list of services available
+#          Examples zones: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
+#          freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
+#          kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
+#          postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
+#          telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
+# Values:  STRING Default: ssh 
+service = ssh
+
+# Option:  rejecttype (ipv4)
+# Note:    See iptables/firewalld man pages for ipv4 reject types. 
+# Values:  STRING
+rejecttype = icmp-port-unreachable
+
+# Option:  blocktype (ipv4/ipv6)
+# Note:    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
+#          REJECT --reject-with icmp-port-unreachable, DROP
+# Values:  STRING
+blocktype = REJECT --reject-with <rejecttype>
+
+# Option:  rich-blocktype (ipv4/ipv6)
+# Note:    See firewalld man pages for jump targets. Common values are reject, 
+#          reject type="icmp-port-unreachable", drop
+# Values:  STRING
+rich-blocktype = reject type='<rejecttype>'
+
+[Init?family=inet6]
+
+# Option:  family(ipv6)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv6
+
+# Option:  rejecttype (ipv6)
+# Note:    See iptables/firewalld man pages for ipv6 reject types. 
+# Values:  STRING
+rejecttype = icmp6-port-unreachable
diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 38b0f3d3..b05f4f53 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -14,14 +14,14 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
 actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
              ipset flush fail2ban-<name>
              ipset destroy fail2ban-<name>
 
diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf
index 438d4cf7..cc40d668 100644
--- a/config/action.d/firewallcmd-multiport.conf
+++ b/config/action.d/firewallcmd-multiport.conf
@@ -5,59 +5,28 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 [Init]
 
-# Default name of the chain
-name = default
-
-chain = INPUT_direct
-
 # Could also use port numbers separated by a comma. 
 port = 1:65535
 
-
-# Option:  protocol
-# Values:  [ tcp | udp | icmp | all ]
-
-protocol = tcp
-
-
-
-# DEV NOTES:
-#
-# Author: Donald Yandt 
-# Uses "FirewallD" instead of the "iptables daemon".
-#
-#
-# Output:
-# actionstart:
-# $ firewall-cmd --direct --add-chain ipv4 filter f2b-apache-modsecurity
-# success
-# $ firewall-cmd --direct --add-rule ipv4 filter f2b-apache-modsecurity 1000 -j RETURN
-# success
-# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 80,443 -j f2b-apache-modsecurity
-# success
-# actioncheck:
-# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
-# f2b-apache-modsecurity
-
diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf
index ac72a68a..e64601e1 100644
--- a/config/action.d/firewallcmd-new.conf
+++ b/config/action.d/firewallcmd-new.conf
@@ -4,32 +4,23 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | grep -q 'f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Option:  chain
-# Notes    specifies the iptables chain to which the fail2ban rules should be
-#          added
-# Values:  [ STRING ]
-#
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #
diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf
index 1b88c2d9..34cb2413 100644
--- a/config/action.d/firewallcmd-rich-logging.conf
+++ b/config/action.d/firewallcmd-rich-logging.conf
@@ -15,6 +15,10 @@
 # firewall-cmd [--zone=<zone>] --list-all
 # firewall-cmd [--zone=zone] --query-rich-rule='rule'
 
+[INCLUDES]
+
+before = firewallcmd-common.conf
+
 [Definition]
 
 actionstart = 
@@ -26,40 +30,22 @@ actioncheck =
 # you can also use zones and/or service names. 
 #
 # zone example: 
-# 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+#
 # service name example:
-#	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+#
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"; done
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"; done
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
 
 [Init]
 
-name = default
-
 # log levels are "emerg", "alert", "crit", "error", "warning", "notice", "info" or "debug"
 level = info
 
 # log rate per minute
 rate = 1
 
-zone = public
-
-# use command firewall-cmd --get-services to see a list of services available
-#
-# Examples:
-#
-# amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
-# freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
-# kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
-# postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
-# telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
-
-service = ssh
-
-# reject types: 'icmp-net-unreachable', 'icmp-host-unreachable', 'icmp-port-unreachable', 'icmp-proto-unreachable', 
-# 'icmp-net-prohibited', 'icmp-host-prohibited', 'icmp-admin-prohibited' or 'tcp-reset'
-
-blocktype = reject type='icmp-port-unreachable'
diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf
index 4e39df54..e64c3823 100644
--- a/config/action.d/firewallcmd-rich-rules.conf
+++ b/config/action.d/firewallcmd-rich-rules.conf
@@ -13,6 +13,10 @@
 # firewall-cmd [--zone=<zone>] --list-all
 # firewall-cmd [--zone=zone] --query-rich-rule='rule'
 
+[INCLUDES]
+
+before = firewallcmd-common.conf
+
 [Definition]
 
 actionstart = 
@@ -24,34 +28,15 @@ actioncheck =
 #you can also use zones and/or service names. 
 #
 # zone example: 
-# 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' <rich-blocktype>"
+#
 # service name example:
-#	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' <rich-blocktype>"
+#
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' <blocktype>"; done
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' <blocktype>"; done
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
 
-[Init]
 
-name = default
-
-zone = public
-
-# use command firewall-cmd --get-services to see a list of services available
-#
-# Examples:
-#
-# amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
-# freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
-# kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
-# postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
-# telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
-
-service = ssh
-
-# reject types: 'icmp-net-unreachable', 'icmp-host-unreachable', 'icmp-port-unreachable', 'icmp-proto-unreachable', 
-# 'icmp-net-prohibited', 'icmp-host-prohibited', 'icmp-admin-prohibited' or 'tcp-reset'
-
-blocktype = reject type='icmp-port-unreachable'

From 810d5996b5b96ed637041492a7f80584973fa47b Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:10:25 -0400
Subject: [PATCH 15/61] Update firewallcmd-rich-logging.conf

---
 config/action.d/firewallcmd-rich-logging.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf
index 34cb2413..d2c8fc2f 100644
--- a/config/action.d/firewallcmd-rich-logging.conf
+++ b/config/action.d/firewallcmd-rich-logging.conf
@@ -30,10 +30,10 @@ actioncheck =
 # you can also use zones and/or service names. 
 #
 # zone example: 
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
 #
 # service name example:
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 

From 07de83e04a453ec5bb172f48db0ba863cf8d42a7 Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:38:10 -0400
Subject: [PATCH 16/61] Update firewallcmd-common.conf

---
 config/action.d/firewallcmd-common.conf | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/config/action.d/firewallcmd-common.conf b/config/action.d/firewallcmd-common.conf
index 106ef216..4abe5318 100644
--- a/config/action.d/firewallcmd-common.conf
+++ b/config/action.d/firewallcmd-common.conf
@@ -10,6 +10,16 @@
 # Values:  STRING
 name = default
 
+# Option   port
+# Notes    Can also use port numbers separated by a comma and in rich-rules comma and/or space.
+# Value    STRING Default: 1:65535
+port = 1:65535
+
+# Option:  protocol
+# Notes    [ tcp | udp | icmp | all ]
+# Values:  STRING Default: tcp
+protocol = tcp
+
 # Option:  family(ipv4)
 # Notes    specifies the socket address family type
 # Values:  STRING  
@@ -28,7 +38,7 @@ zone = public
 
 # Option:  service
 # Notes    use command firewall-cmd --get-services to see a list of services available
-#          Examples zones: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
+#          Examples services: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
 #          freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
 #          kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
 #          postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
@@ -37,18 +47,18 @@ zone = public
 service = ssh
 
 # Option:  rejecttype (ipv4)
-# Note:    See iptables/firewalld man pages for ipv4 reject types. 
+# Notes    See iptables/firewalld man pages for ipv4 reject types. 
 # Values:  STRING
 rejecttype = icmp-port-unreachable
 
 # Option:  blocktype (ipv4/ipv6)
-# Note:    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
+# Notes    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
 #          REJECT --reject-with icmp-port-unreachable, DROP
 # Values:  STRING
 blocktype = REJECT --reject-with <rejecttype>
 
 # Option:  rich-blocktype (ipv4/ipv6)
-# Note:    See firewalld man pages for jump targets. Common values are reject, 
+# Notes    See firewalld man pages for jump targets. Common values are reject, 
 #          reject type="icmp-port-unreachable", drop
 # Values:  STRING
 rich-blocktype = reject type='<rejecttype>'

From ffebde68e0fae17b07e620b8d9bc62aa4c967426 Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:38:36 -0400
Subject: [PATCH 17/61] Update firewallcmd-multiport.conf

---
 config/action.d/firewallcmd-multiport.conf | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf
index cc40d668..81540e5b 100644
--- a/config/action.d/firewallcmd-multiport.conf
+++ b/config/action.d/firewallcmd-multiport.conf
@@ -24,9 +24,3 @@ actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\
 actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Could also use port numbers separated by a comma. 
-port = 1:65535
-

From 0c44ecfc770113b2e0c2ff3937d3e3a91d7a97ca Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 14 May 2016 15:00:23 +0200
Subject: [PATCH 18/61] action.d/firewallcmd-ipset.conf: different name of the
 match set's for IPv4/IPv6, using conditional <ipmset>, analog to the
 iptables-ipset; test cases for 3 firewallcmd extended;

---
 config/action.d/firewallcmd-ipset.conf |  20 +++--
 fail2ban/tests/servertestcase.py       | 108 +++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 7 deletions(-)

diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index b05f4f53..69447627 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -18,16 +18,16 @@ before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
-             ipset flush fail2ban-<name>
-             ipset destroy fail2ban-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
-actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
-actionunban = ipset del fail2ban-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -44,6 +44,12 @@ chain = INPUT_direct
 
 bantime = 600
 
+ipmset = f2b-<name>
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+
 
 # DEV NOTES:
 #
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index cc14e97f..1809933c 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1391,6 +1391,114 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
+				# firewallcmd-multiport --
+				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
+					'start': (
+						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`",
+					),
+					'ip4-check': (
+						r"`firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-mp$'`",
+					),
+					'ip6-check': (
+						r"`firewall-cmd --direct --get-chains ipv6 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-mp$'`",
+					),
+					'ip4-ban': (
+						r"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv4 filter f2b-j-w-fwcmd-mp 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv6 filter f2b-j-w-fwcmd-mp 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# firewallcmd-allports --
+				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
+					'start': (
+						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-ap`",
+					),
+					'ip4-check': (
+						r"`firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-ap$'`",
+					),
+					'ip6-check': (
+						r"`firewall-cmd --direct --get-chains ipv6 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-ap$'`",
+					),
+					'ip4-ban': (
+						r"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv4 filter f2b-j-w-fwcmd-ap 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv6 filter f2b-j-w-fwcmd-ap 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# firewallcmd-ipset --
+				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
+					'start': (
+						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset6`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`",
+					),					
+				}),
 			)
 			server = TestServer()
 			transm = server._Server__transm

From 975608dfb629c637622f0d638335637c1c688970 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 15 May 2016 21:08:32 +0200
Subject: [PATCH 19/61] no hardcoded python interpreter path

---
 bin/fail2ban-client                                 | 2 +-
 bin/fail2ban-regex                                  | 2 +-
 bin/fail2ban-server                                 | 2 +-
 bin/fail2ban-testcases                              | 2 +-
 config/filter.d/ignorecommands/apache-fakegooglebot | 2 +-
 fail2ban/client/fail2banregex.py                    | 2 +-
 fail2ban/tests/files/config/apache-auth/digest.py   | 2 +-
 fail2ban/tests/files/ignorecommand.py               | 2 +-
 setup.py                                            | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/bin/fail2ban-client b/bin/fail2ban-client
index bc0c0be8..4b1407c4 100755
--- a/bin/fail2ban-client
+++ b/bin/fail2ban-client
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 
diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex
index 584c1ea7..09044f0a 100755
--- a/bin/fail2ban-regex
+++ b/bin/fail2ban-regex
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/bin/fail2ban-server b/bin/fail2ban-server
index f522f418..5ec645a4 100755
--- a/bin/fail2ban-server
+++ b/bin/fail2ban-server
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 
diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 606b0b06..768c584d 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 """Script to run Fail2Ban tests battery
diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index 86a28eaa..fe4b6591 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
 #
 # Written in Python to reuse built-in Python batteries and not depend on
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 9ecb7229..fae4f354 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py
index 875ebffe..f4fcfcb9 100755
--- a/fail2ban/tests/files/config/apache-auth/digest.py
+++ b/fail2ban/tests/files/config/apache-auth/digest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 import requests
 
 try:
diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py
index dd6b5aab..473980ec 100755
--- a/fail2ban/tests/files/ignorecommand.py
+++ b/fail2ban/tests/files/ignorecommand.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 import sys
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
diff --git a/setup.py b/setup.py
index e3c499d2..5579c981 100755
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 

From de813acf5104dc00c0086b544c6fe82f428d4ca8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 11:33:49 +0200
Subject: [PATCH 20/61] extends generic `__prefix_line` with optional brackets
 for the date ambit (gh-1421), added new parameter `__date_ambit` + test case
 added;

---
 ChangeLog                                     | 2 ++
 config/filter.d/common.conf                   | 4 +++-
 fail2ban/tests/files/logs/zzz-generic-example | 3 +++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 44ba9d11..d52e1cff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -21,6 +21,8 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * filter.d/common.conf
      - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
      - all optional spaces normalized in `common.conf`, test covered now
+     - generic `__prefix_line` extended with optional brackets for the date ambit (gh-1421),
+       added new parameter `__date_ambit`
 
 - New Features:
    * New Actions:
diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 115fa96c..23fd1d5a 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -46,12 +46,14 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
 __bsd_syslog_verbose = <[^.]+\.[^.]+>
 
+__date_ambit = \[\]
+
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
+__prefix_line = (?:%(__date_ambit)s)?\s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index ecc5a1c6..e3480b63 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -26,3 +26,6 @@ Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-de
 # -- the same as above with additional spaces around --
 # failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
 Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3
+# -- the same as above with brackets as date ambit --
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
+[Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3

From cb4f9be8b2748e940303cff77d72ab7fdcf06a53 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 11:54:08 +0200
Subject: [PATCH 21/61] the date brackets removed from filters using
 `__prefix_line`, because `__prefix_line` already contains the date ambit;

---
 config/filter.d/asterisk.conf | 20 ++++++++++----------
 config/filter.d/nsd.conf      |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 3975fb29..01063efa 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -16,17 +16,17 @@ __pid_re = (?:\[\d+\])
 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 
 # All Asterisk log messages begin like this:
-log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)?
+log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*( in \w+:)?
 
-failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
-            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )Ext\. s: "Rejecting unknown SIP connection from <HOST>"$
+failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+            ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
+            ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
+            ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
+            ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
 
 ignoreregex =
 
diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf
index 70b41ca4..8f32f7be 100644
--- a/config/filter.d/nsd.conf
+++ b/config/filter.d/nsd.conf
@@ -22,7 +22,7 @@ _daemon = nsd
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 
-failregex =  ^\[\]%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
-              ^\[\]%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
+failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
+             ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =

From 5e06e8be624b1d46169c60797b6959f0c9d7ee49 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 16:42:23 +0200
Subject: [PATCH 22/61] removed shebang and changed mode for fail2banregex.py

---
 fail2ban/client/fail2banregex.py | 1 -
 1 file changed, 1 deletion(-)
 mode change 100755 => 100644 fail2ban/client/fail2banregex.py

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
old mode 100755
new mode 100644
index fae4f354..e2222792
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #

From c6f63c7263741403d24faaf4eb59f716fea205c5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 18:06:46 +0200
Subject: [PATCH 23/61] jailreader: support multiple option groups, syntax
 `action = act[p1=...][p2=...]` + test case for it (see gh-1425, gh-1429)

---
 fail2ban/client/jailreader.py          | 7 ++++++-
 fail2ban/tests/clientreadertestcase.py | 7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index fda5d40c..98086cf4 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -42,9 +42,14 @@ logSys = getLogger(__name__)
 
 class JailReader(ConfigReader):
 	
+	# regex, to extract list of options:
 	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
+	# regex, to iterate over single option in option list, syntax:
+	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
+	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
+	# `action = act[p1=...][p2=...]`
 	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
+		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)')
 	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 5e66e25e..521fda2e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -255,6 +255,13 @@ class JailReaderTest(LogCaptureTestCase):
 		result = JailReader.extractOptions(option)
 		self.assertEqual(expected, result)
 
+		# And multiple groups (`][` instead of `,`)
+		result = JailReader.extractOptions(option.replace(',', ']['))
+		expected2 = (expected[0],
+		 dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems())
+		)
+		self.assertEqual(expected2, result)
+
 	def testVersionAgent(self):
 		jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR)
 		# emulate jail.read(), because such jail not exists:

From 3e1328c83b2d6fe9b444df1b13a74738a3c78123 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 15 May 2016 22:30:07 +0200
Subject: [PATCH 24/61] split PF config files between all- and multi port

---
 config/action.d/pf-allports.conf              | 73 +++++++++++++++++++
 .../action.d/{pf.conf => pf-multiport.conf}   |  6 --
 2 files changed, 73 insertions(+), 6 deletions(-)
 create mode 100644 config/action.d/pf-allports.conf
 rename config/action.d/{pf.conf => pf-multiport.conf} (95%)

diff --git a/config/action.d/pf-allports.conf b/config/action.d/pf-allports.conf
new file mode 100644
index 00000000..e77bea43
--- /dev/null
+++ b/config/action.d/pf-allports.conf
@@ -0,0 +1,73 @@
+# Fail2Ban configuration file
+#
+# OpenBSD pf ban/unban
+#
+# Author: Nick Hilliard <nick@foobar.org>
+# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
+#
+#
+
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+# we don't enable PF automatically; to enable run pfctl -e 
+# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to any" | pfctl -f-
+
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+# we only disable PF rules we've installed prior
+actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
+             pfctl -t <tablename>-<name> -T flush
+             pfctl -t <tablename>-<name> -T kill
+
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck = pfctl -sr | grep -q <tablename>-<name>
+
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+actionban = pfctl -t <tablename>-<name> -T add <ip>
+
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+# note -r option used to remove matching rule
+actionunban = pfctl -t <tablename>-<name> -T delete <ip>
+
+[Init]
+# Option:  tablename
+# Notes.:  The pf table name.
+# Values:  [ STRING ]
+#
+tablename = f2b
+
+# Option:  protocol
+# Notes.:  internally used by config reader for interpolations.
+# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
+#
+protocol = tcp
+
diff --git a/config/action.d/pf.conf b/config/action.d/pf-multiport.conf
similarity index 95%
rename from config/action.d/pf.conf
rename to config/action.d/pf-multiport.conf
index effd080c..6814a9f3 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf-multiport.conf
@@ -71,9 +71,3 @@ tablename = f2b
 #
 protocol = tcp
 
-# Option:  port
-# Notes.:  the port to block, defaults to any
-# Values:  [ STRING ]
-#
-port = any
-

From e74047ae49807a59d06ccfcc48350c2cab35c59a Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 16:27:48 +0200
Subject: [PATCH 25/61] revert to common config for PF covering multi and
 allports

---
 config/action.d/pf-multiport.conf             | 73 -------------------
 config/action.d/{pf-allports.conf => pf.conf} | 11 ++-
 2 files changed, 10 insertions(+), 74 deletions(-)
 delete mode 100644 config/action.d/pf-multiport.conf
 rename config/action.d/{pf-allports.conf => pf.conf} (91%)

diff --git a/config/action.d/pf-multiport.conf b/config/action.d/pf-multiport.conf
deleted file mode 100644
index 6814a9f3..00000000
--- a/config/action.d/pf-multiport.conf
+++ /dev/null
@@ -1,73 +0,0 @@
-# Fail2Ban configuration file
-#
-# OpenBSD pf ban/unban
-#
-# Author: Nick Hilliard <nick@foobar.org>
-# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
-#
-#
-
-[Definition]
-
-# Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
-# Values:  CMD
-#
-# we don't enable PF automatically; to enable run pfctl -e 
-# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
-actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any port <port>" | pfctl -f-
-
-
-# Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
-# Values:  CMD
-#
-# we only disable PF rules we've installed prior
-actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
-             pfctl -t <tablename>-<name> -T flush
-             pfctl -t <tablename>-<name> -T kill
-
-
-# Option:  actioncheck
-# Notes.:  command executed once before each actionban command
-# Values:  CMD
-#
-actioncheck = pfctl -sr | grep -q <tablename>-<name>
-
-
-# Option:  actionban
-# Notes.:  command executed when banning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
-# Values:  CMD
-#
-actionban = pfctl -t <tablename>-<name> -T add <ip>
-
-
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
-# Values:  CMD
-#
-# note -r option used to remove matching rule
-actionunban = pfctl -t <tablename>-<name> -T delete <ip>
-
-[Init]
-# Option:  tablename
-# Notes.:  The pf table name.
-# Values:  [ STRING ]
-#
-tablename = f2b
-
-# Option:  protocol
-# Notes.:  internally used by config reader for interpolations.
-# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
-#
-protocol = tcp
-
diff --git a/config/action.d/pf-allports.conf b/config/action.d/pf.conf
similarity index 91%
rename from config/action.d/pf-allports.conf
rename to config/action.d/pf.conf
index e77bea43..10b0f328 100644
--- a/config/action.d/pf-allports.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to any <actionoptions>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -71,3 +71,12 @@ tablename = f2b
 #
 protocol = tcp
 
+
+# Option:  port
+# Notes.:  specifies port to monitor
+# Values:  [ NUM | STRING ]  Default:
+#
+port = telnet
+
+actionoptions = 
+multiport = port <port>

From 25af11215b71dec2197b8df9ee1aa9f813965aca Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 20:08:46 +0200
Subject: [PATCH 26/61] test case for generic common moved to
 `./fail2ban/tests/config/filter.d/zzz-generic-example.conf` to prevent
 shipping it with fail2ban installations

---
 fail2ban/tests/clientreadertestcase.py        |  7 +++---
 .../config}/filter.d/zzz-generic-example.conf |  2 +-
 fail2ban/tests/samplestestcase.py             | 25 +++++++++++--------
 3 files changed, 19 insertions(+), 15 deletions(-)
 rename {config => fail2ban/tests/config}/filter.d/zzz-generic-example.conf (85%)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index b5c31c38..0a3734e5 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -600,10 +600,9 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
-			filters = (os.path.splitext(os.path.split(flt)[1])[0]
-				for flt in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not flt.endswith('common.conf'))
-			filters = set(filter(lambda flt: not flt.startswith('zzz-'), filters))
+			filters = set(os.path.splitext(os.path.split(a)[1])[0]
+				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+					if not a.endswith('common.conf'))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
diff --git a/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
similarity index 85%
rename from config/filter.d/zzz-generic-example.conf
rename to fail2ban/tests/config/filter.d/zzz-generic-example.conf
index 421c117d..a59ccb1e 100644
--- a/config/filter.d/zzz-generic-example.conf
+++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
@@ -7,7 +7,7 @@
 
 # Read common prefixes. If any customizations available -- read them from
 # common.local
-before = common.conf
+before = ../../../../config/filter.d/common.conf
 
 [Definition]
 
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 9e6c0ee7..a40a11da 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -35,6 +35,7 @@ from ..server.filter import Filter
 from ..client.filterreader import FilterReader
 from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR
 
+TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 
@@ -60,11 +61,11 @@ class FilterSamplesRegex(unittest.TestCase):
 			"Expected more FilterSampleRegexs tests")
 
 
-def testSampleRegexsFactory(name):
+def testSampleRegexsFactory(name, basedir):
 	def testFilter(self):
 
 		# Check filter exists
-		filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR)
+		filterConf = FilterReader(name, "jail", {}, basedir=basedir)
 		self.assertEqual(filterConf.getFile(), name)
 		self.assertEqual(filterConf.getJailName(), "jail")
 		filterConf.read()
@@ -147,11 +148,15 @@ def testSampleRegexsFactory(name):
 
 	return testFilter
 
-for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'),
-					  os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
-	filterName = filter_.rpartition(".")[0]
-	if not filterName.startswith('.'):
-		setattr(
-			FilterSamplesRegex,
-			"testSampleRegexs%s" % filterName.upper(),
-			testSampleRegexsFactory(filterName))
+for basedir_, filter_ in (
+	(CONFIG_DIR, lambda x: not x.endswith('common.conf') and x.endswith('.conf')),
+	(TEST_CONFIG_DIR, lambda x: x.startswith('zzz-') and x.endswith('.conf')),
+):
+	for filter_ in filter(filter_,
+						  os.listdir(os.path.join(basedir_, "filter.d"))):
+		filterName = filter_.rpartition(".")[0]
+		if not filterName.startswith('.'):
+			setattr(
+				FilterSamplesRegex,
+				"testSampleRegexs%s" % filterName.upper(),
+				testSampleRegexsFactory(filterName, basedir_))

From 1e7fd26f5f42fb9eba0fce8bfc54e8e128bae3e7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 20:51:12 +0200
Subject: [PATCH 27/61] rename `actionoptions` to `actiontype` in pf-action
 (multiport) + fixed test cases

---
 config/action.d/pf.conf          |  6 +++---
 fail2ban/tests/servertestcase.py | 23 +++++++++++++++++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 10b0f328..1312538e 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any <actionoptions>" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to any<actiontype>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -78,5 +78,5 @@ protocol = tcp
 #
 port = telnet
 
-actionoptions = 
-multiport = port <port>
+actiontype = 
+multiport = <sp>port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 1809933c..ac9927bb 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1372,12 +1372,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
 					),					
 				}),
-				# pf --
+				# pf allports --
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1391,6 +1391,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
+				# pf multiport --
+				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf-mp -T flush`',
+						'`pfctl -t f2b-j-w-pf-mp -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
+				}),
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),

From b5e031f3c3d35d75ea2821ef6a57b191c7796964 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 21:19:34 +0200
Subject: [PATCH 28/61] some documentation for multiport use in pf.conf

---
 config/action.d/pf.conf | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 1312538e..5710eb8e 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -78,5 +78,12 @@ protocol = tcp
 #
 port = telnet
 
+# Option: actiontype
+# Notes.: defines additions to the blocking rule
+# Values: leave empty to block all attempts from the host
 actiontype = 
+
+# Option: multiport
+# Notes.: addition to block access only to specific ports
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
 multiport = <sp>port <port>

From 4d51c591c1f3ce39b407d85c2f14ce3a132360fb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 14:50:41 +0200
Subject: [PATCH 29/61] pf.conf: warranted consistently echoing for the pf
 actiontype if actiontype or multiport tags will be customized;

---
 config/action.d/pf.conf | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 5710eb8e..ea70d894 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any<actiontype>" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -76,14 +76,20 @@ protocol = tcp
 # Notes.:  specifies port to monitor
 # Values:  [ NUM | STRING ]  Default:
 #
-port = telnet
+#port = telnet
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
 # Values: leave empty to block all attempts from the host
-actiontype = 
+# Default: Value of the allports
+actiontype = any
+
+# Option: allports
+# Notes.: default addition to block all ports
+# Usage.: use in jail config: "banaction = pf[actiontype=<allports>]"
+allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
-multiport = <sp>port <port>
+multiport = any port <port>

From 1ebc3facb1f30c2269f45284aa9072a84be078ba Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 May 2016 10:34:28 -0400
Subject: [PATCH 30/61] BF: maintain previous default beh for pf -- ban a port
 (ssh) only

---
 config/action.d/pf.conf          |  9 ++++-----
 fail2ban/tests/servertestcase.py | 27 +++++++++++++++++++++++----
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index ea70d894..40f4620a 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -75,14 +75,13 @@ protocol = tcp
 # Option:  port
 # Notes.:  specifies port to monitor
 # Values:  [ NUM | STRING ]  Default:
-#
-#port = telnet
+port = ssh
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
 # Values: leave empty to block all attempts from the host
-# Default: Value of the allports
-actiontype = any
+# Default: Value of the multiport
+actiontype = <multiport>
 
 # Option: allports
 # Notes.: default addition to block all ports
@@ -91,5 +90,5 @@ allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
-# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>, port="%(port)s"]"
 multiport = any port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ac9927bb..5fbf8cd6 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1370,14 +1370,14 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),
 					'ip6-unban': (
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
-					),					
+					),
 				}),
-				# pf allports --
+				# pf default - multiport on ssh --
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port ssh" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1391,7 +1391,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
-				# pf multiport --
+				# pf multiport with custom port --
 				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
 					'ip4': (), 'ip6': (),
 					'start': (
@@ -1410,6 +1410,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
+				# pf allports --
+				('j-w-pf-ap', 'pf[name=%(__name__)s,actiontype=<allports>]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf-ap -T flush`',
+						'`pfctl -t f2b-j-w-pf-ap -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 2001:db8::`",),
+				}),
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),

From 0fdc56546f8391760a5c7af0ef32749b76f8d091 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 17:45:41 +0200
Subject: [PATCH 31/61] Fixed misunderstanding of port in (ban)action: port
 will be always specified in jail config ([DEFAULT] or jail)

---
 config/action.d/pf.conf          | 6 +-----
 fail2ban/tests/servertestcase.py | 6 +++---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 40f4620a..b7476fa2 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -72,10 +72,6 @@ tablename = f2b
 protocol = tcp
 
 
-# Option:  port
-# Notes.:  specifies port to monitor
-# Values:  [ NUM | STRING ]  Default:
-port = ssh
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
@@ -90,5 +86,5 @@ allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
-# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>, port="%(port)s"]"
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
 multiport = any port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 5fbf8cd6..fabf77b9 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1372,12 +1372,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
 					),
 				}),
-				# pf default - multiport on ssh --
+				# pf default -- multiport on default port (tag <port> set in jail.conf, but not in this test case)
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port ssh" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1411,7 +1411,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
 				# pf allports --
-				('j-w-pf-ap', 'pf[name=%(__name__)s,actiontype=<allports>]', {
+				('j-w-pf-ap', 'pf[actiontype=<allports>][name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',

From 52377984cd415c089438bdfbcc2c86e344f8ca59 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 17:01:44 +0200
Subject: [PATCH 32/61] back to mandatory space, ungrouping of sub parameters
 in `__prefix_line` + small code review;

---
 config/filter.d/common.conf | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 23fd1d5a..586f428a 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -30,7 +30,7 @@ __daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = %(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -46,14 +46,16 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
 __bsd_syslog_verbose = <[^.]+\.[^.]+>
 
-__date_ambit = \[\]
+__vserver = @vserver_\S+
+
+__date_ambit = (?:\[\])
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = (?:%(__date_ambit)s)?\s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
+__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap

From 932708de9e6a3ba5089116cc3c1ee456ba3b6855 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 19:03:32 +0200
Subject: [PATCH 33/61] fixed --pidfile bug, introduced in gh-1322:
 gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon,
 not argument of fail2ban (see gh-1434) closes gh-1434

---
 ChangeLog          | 2 ++
 files/gentoo-initd | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9ac9a752..55fb8aff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -18,6 +18,8 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, 
+     not argument of fail2ban (see gh-1434)
 
 - New Features:
    * New Actions:
diff --git a/files/gentoo-initd b/files/gentoo-initd
index b7e1067b..c5b5b702 100755
--- a/files/gentoo-initd
+++ b/files/gentoo-initd
@@ -34,15 +34,15 @@ start() {
 	# remove stalled sock file after system crash
 	# bug 347477
 	rm -f /var/run/fail2ban/fail2ban.sock || return 1
-	start-stop-daemon --start --exec ${FAIL2BAN} start \
-	--pidfile /var/run/fail2ban/fail2ban.pid
+	start-stop-daemon --start --pidfile /var/run/fail2ban/fail2ban.pid \
+	-- ${FAIL2BAN} start
 	eend $? "Failed to start fail2ban"
 }
 
 stop() {
 	ebegin "Stopping fail2ban"
-	start-stop-daemon --stop --exec ${FAIL2BAN} stop \
-	--pidfile /var/run/fail2ban/fail2ban.pid
+	start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid \
+	-- ${FAIL2BAN} stop
 	eend $? "Failed to stop fail2ban"
 }
 

From db9e7240386b7fdee5d796138f7118d01e3168e0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 12:03:35 +0200
Subject: [PATCH 34/61] extremely speedup of all database operations: -
 (synchronous = OFF) write data through OS without syncing - (journal_mode =
 MEMORY) use memory for the transaction logging

---
 fail2ban/server/database.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 7de87554..c0e9bf46 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -182,8 +182,11 @@ class Fail2BanDb(object):
 			raise
 
 		cur = self._db.cursor()
-		cur.execute("PRAGMA foreign_keys = ON;")
-
+		cur.execute("PRAGMA foreign_keys = ON")
+		# speedup: write data through OS without syncing (no wait):
+		cur.execute("PRAGMA synchronous = OFF")
+		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
+		cur.execute("PRAGMA journal_mode = MEMORY")
 		try:
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:

From baafac36a4c578769d53240147d2777b6414ad1c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 12:42:18 +0200
Subject: [PATCH 35/61] ChangeLog entry

---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 433026f3..76719f16 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * New Actions:
      - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
 - Enhancements:
+   * Extreme speedup of all sqlite database operations (gh-1436),
+     by using of following sqlite options:
+     - (synchronous = OFF) write data through OS without syncing
+     - (journal_mode = MEMORY) use memory for the transaction logging
+     - (temp_store = MEMORY) temporary tables and indices are kept in memory
    * journald journalmatch for pure-ftpd (gh-1362)
    * Add additional regex filter for dovecot ldap authentication failures (gh-1370)
    * added additional regex filters for exim (gh-1371)

From 1718c8dbe92030617042db2c7a01e996cf16cab7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 20 May 2016 13:26:51 +0200
Subject: [PATCH 36/61] pypy: switch journal mode after upgrade (save it during
 the upgrade), to prevent errors like "database table is locked"

---
 fail2ban/server/database.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index c0e9bf46..560fbfe5 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -181,12 +181,24 @@ class Fail2BanDb(object):
 				filename, e.args[0])
 			raise
 
+		# differentiate pypy: switch journal mode later (save it during the upgrade), 
+		# to prevent errors like "database table is locked":
+		try:
+			import __pypy__
+			pypy = True
+		except ImportError:
+			pypy = False
+
 		cur = self._db.cursor()
 		cur.execute("PRAGMA foreign_keys = ON")
 		# speedup: write data through OS without syncing (no wait):
 		cur.execute("PRAGMA synchronous = OFF")
 		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
-		cur.execute("PRAGMA journal_mode = MEMORY")
+		if not pypy:
+			cur.execute("PRAGMA journal_mode = MEMORY")
+		# speedup: temporary tables and indices are kept in memory:
+		cur.execute("PRAGMA temp_store = MEMORY")
+
 		try:
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:
@@ -205,6 +217,9 @@ class Fail2BanDb(object):
 						Fail2BanDb.__version__, version, newversion)
 					raise RuntimeError('Failed to fully update')
 		finally:
+			# pypy: set journal mode after possible upgrade db:
+			if pypy:
+				cur.execute("PRAGMA journal_mode = MEMORY")
 			cur.close()
 
 	@property
@@ -247,13 +262,14 @@ class Fail2BanDb(object):
 
 		A timestamped backup is also created prior to attempting the update.
 		"""
-		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
-		shutil.copyfile(self.filename, self._dbBackupFilename)
-		logSys.info("Database backup created: %s", self._dbBackupFilename)
 		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
 						"Attempt to travel to future version of database ...how did you get here??")
 
+		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+		shutil.copyfile(self.filename, self._dbBackupFilename)
+		logSys.info("Database backup created: %s", self._dbBackupFilename)
+
 		if version < 2:
 			cur.executescript("BEGIN TRANSACTION;"
 						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"

From 3d99833570d9768c9c2d050fa143f5cae38d7dd4 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 21 May 2016 13:19:49 +0200
Subject: [PATCH 37/61] forgotten convert for `bantime` (str2seconds) in
 action, as long as it should be supplied as parameter of action executable;
 covered via test cases now

---
 fail2ban/server/action.py        |  4 ++--
 fail2ban/tests/servertestcase.py | 24 ++++++++++++------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index ba3d4633..51c5e077 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -229,8 +229,8 @@ class CommandAction(ActionBase):
 	def __setattr__(self, name, value):
 		if not name.startswith('_') and not callable(value):
 			# special case for some pasrameters:
-			if name == 'timeout':
-				value = MyTime.str2seconds(value)
+			if name in ('timeout', 'bantime'):
+				value = str(MyTime.str2seconds(value))
 			# parameters changed - clear properties and substitution cache:
 			self.__properties = None
 			self.__substCache.clear()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ac9927bb..d39847ca 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -638,11 +638,11 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(
 			self.transm.proceed(
 				["set", self.jailName, "action", action, "timeout", "10"]),
-			(0, 10))
+			(0, "10"))
 		self.assertEqual(
 			self.transm.proceed(
 				["get", self.jailName, "action", action, "timeout"]),
-			(0, 10))
+			(0, "10"))
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "delaction", action]),
 			(0, None))
@@ -1125,7 +1125,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			#   etc.
 			testJailsActions = (
 				# iptables-multiport --
-				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
@@ -1163,7 +1163,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-allports --
-				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
@@ -1201,7 +1201,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6 --
-				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
@@ -1233,7 +1233,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6-allports --
-				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
@@ -1265,7 +1265,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables --
-				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables`",
@@ -1303,7 +1303,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-new --
-				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-new`",
@@ -1341,7 +1341,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-xt_recent-echo --
-				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
 					'start': (
 						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
@@ -1411,7 +1411,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
 				# firewallcmd-multiport --
-				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
@@ -1449,7 +1449,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-allports --
-				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
@@ -1487,7 +1487,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-ipset --
-				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",

From 743a531eb59a4f3704dda68557fb34481538e159 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 21 May 2016 10:08:54 -0400
Subject: [PATCH 38/61] BF: make :port and I=[ip]:port optional for a "AUTH
 command used when not advertised"

Closes #1430
---
 config/filter.d/exim.conf      | 2 +-
 fail2ban/tests/files/logs/exim | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 4aadf15c..54ad20d8 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ \[<HOST>\](:\d+)? (I=\[\S+\](:\d+)? )?dropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" H=(|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ AUTH command used when not advertised\s*$
+            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" H=(|\S* )(|\(\S*\) )\[<HOST>\](?:\:\d+)? (?:I=\[\S*\]\:\d+ )?AUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ D=\d+s(| C=\S*)\s*$
             ^%(pid)s \S+ SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ closed by DROP in ACL\s*$
 
diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index a3b287d4..4b3a7ff5 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -48,6 +48,8 @@
 2016-03-18 00:34:06 [7513] SMTP protocol error in "AUTH LOGIN" H=(ylmf-pc) [45.32.34.167]:60723 I=[172.89.0.6]:587 AUTH command used when not advertised
 # failJSON: { "time": "2016-03-19T18:40:44", "match": true , "host": "92.45.204.170" }
 2016-03-19 18:40:44 [26221] SMTP protocol error in "AUTH LOGIN aW5mb0BtYW5iYXQub3Jn" H=([127.0.0.1]) [92.45.204.170]:14243 I=[172.89.0.6]:587 AUTH command used when not advertised
+# failJSON: { "time": "2016-05-17T06:25:27", "match": true , "host": "69.10.61.61", "desc": "from gh-1430" }
+2016-05-17 06:25:27 SMTP protocol error in "AUTH LOGIN" H=(ylmf-pc) [69.10.61.61] AUTH command used when not advertised
 # failJSON: { "time": "2016-03-21T06:38:05", "match": true , "host": "49.212.207.15" }
 2016-03-21 06:38:05 [5718] no MAIL in SMTP connection from www3005.sakura.ne.jp [49.212.207.15]:28890 I=[172.89.0.6]:25 D=21s C=EHLO,STARTTLS
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }

From 8b8cf2a660b5727d2a864b61917d4279f80c1b60 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 21 May 2016 10:27:16 -0400
Subject: [PATCH 39/61] ENH: exim filters -- make more use of %(host_info)s
 which in turn made more flexible

---
 config/filter.d/exim-common.conf | 2 +-
 config/filter.d/exim.conf        | 8 ++++----
 fail2ban/tests/files/logs/exim   | 2 ++
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf
index 1c0a0a20..8f9553fe 100644
--- a/config/filter.d/exim-common.conf
+++ b/config/filter.d/exim-common.conf
@@ -9,7 +9,7 @@ after = exim-common.local
 
 [Definition]
 
-host_info = H=([\w.-]+ )?(\(\S+\) )?\[<HOST>\](:\d+)? (I=\[\S+\]:\d+ )?(U=\S+ )?(P=e?smtp )?
+host_info = (H=([\w.-]+ )?(\(\S+\) )?)?\[<HOST>\](:\d+)? (I=\[\S+\](:\d+)? )?(U=\S+ )?(P=e?smtp )?
 pid = ( \[\d+\])?
 
 # DEV Notes:
diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 54ad20d8..517e3de7 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -17,10 +17,10 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[<HOST>\](:\d+)?( I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
             ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
-            ^%(pid)s SMTP call from \S+ \[<HOST>\](:\d+)? (I=\[\S+\](:\d+)? )?dropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" H=(|\S* )(|\(\S*\) )\[<HOST>\](?:\:\d+)? (?:I=\[\S*\]\:\d+ )?AUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ D=\d+s(| C=\S*)\s*$
-            ^%(pid)s \S+ SMTP connection from (|\S* )(|\(\S*\) )\[<HOST>\]\:\d+ I=\[\S*\]\:\d+ closed by DROP in ACL\s*$
+            ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
+            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s no MAIL in SMTP connection from (|\S* )(|\(\S*\) )%(host_info)sD=\d+s(| C=\S*)\s*$
+            ^%(pid)s \S+ SMTP connection from (|\S* )(|\(\S*\) )%(host_info)sclosed by DROP in ACL\s*$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index 4b3a7ff5..9da5ff22 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -54,6 +54,8 @@
 2016-03-21 06:38:05 [5718] no MAIL in SMTP connection from www3005.sakura.ne.jp [49.212.207.15]:28890 I=[172.89.0.6]:25 D=21s C=EHLO,STARTTLS
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
 2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=10s
+# failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
+2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=10s
 # failJSON: { "time": "2016-03-21T04:07:49", "match": true , "host": "174.137.147.204" }
 2016-03-21 04:07:49 [25874] 1ahr79-0006jK-G9 SMTP connection from (voyeur.webair.com) [174.137.147.204]:44884 I=[172.89.0.6]:25 closed by DROP in ACL
 # failJSON: { "time": "2016-03-21T04:33:13", "match": true , "host": "206.214.71.53" }

From 01d0506ea07af3ad0208743e7d80646349c08d59 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 21 May 2016 10:55:27 -0400
Subject: [PATCH 40/61] ENH: splitcommaspace -> splitwords allow to split
 ignoreip entries with new lines

Closes #1432
---
 fail2ban/client/jailreader.py  |  4 ++--
 fail2ban/helpers.py            |  6 +++---
 fail2ban/tests/misctestcase.py | 18 ++++++++++--------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 56b8889c..c86c3153 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -34,7 +34,7 @@ from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..version import version
 from ..helpers import getLogger
-from ..helpers import splitcommaspace
+from ..helpers import splitwords
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -214,7 +214,7 @@ class JailReader(ConfigReader):
 			elif opt == "maxretry":
 				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
 			elif opt == "ignoreip":
-				for ip in splitcommaspace(self.__opts[opt]):
+				for ip in splitwords(self.__opts[opt]):
 					stream.append(["set", self.__name, "addignoreip", ip])
 			elif opt == "findtime":
 				stream.append(["set", self.__name, "findtime", self.__opts[opt]])
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 8e1b0e32..fc51a88e 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -128,12 +128,12 @@ def excepthook(exctype, value, traceback):
 		"Unhandled exception in Fail2Ban:", exc_info=True)
 	return sys.__excepthook__(exctype, value, traceback)
 
-def splitcommaspace(s):
-	"""Helper to split on any comma or space
+def splitwords(s):
+	"""Helper to split words on any comma, space, or a new line
 
 	Returns empty list if input is empty (or None) and filters
 	out empty entries
 	"""
 	if not s:
 		return []
-	return filter(bool, re.split('[ ,]', s))
+	return filter(bool, map(str.strip, re.split('[ ,\n]', s)))
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index e28ce422..48074d53 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -33,7 +33,7 @@ from glob import glob
 from StringIO import StringIO
 
 from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger
-from ..helpers import splitcommaspace
+from ..helpers import splitwords
 from ..server.datetemplate import DatePatternRegex
 
 
@@ -56,13 +56,15 @@ class HelpersTest(unittest.TestCase):
 			# might be fragile due to ' vs "
 			self.assertEqual(args, "('Very bad', None)")
 
-	def testsplitcommaspace(self):
-		self.assertEqual(splitcommaspace(None), [])
-		self.assertEqual(splitcommaspace(''), [])
-		self.assertEqual(splitcommaspace('  '), [])
-		self.assertEqual(splitcommaspace('1'), ['1'])
-		self.assertEqual(splitcommaspace(' 1 2 '), ['1', '2'])
-		self.assertEqual(splitcommaspace(' 1, 2 , '), ['1', '2'])
+	def testsplitwords(self):
+		self.assertEqual(splitwords(None), [])
+		self.assertEqual(splitwords(''), [])
+		self.assertEqual(splitwords('  '), [])
+		self.assertEqual(splitwords('1'), ['1'])
+		self.assertEqual(splitwords(' 1 2 '), ['1', '2'])
+		self.assertEqual(splitwords(' 1, 2 , '), ['1', '2'])
+		self.assertEqual(splitwords(' 1\n  2'), ['1', '2'])
+		self.assertEqual(splitwords(' 1\n  2, 3'), ['1', '2', '3'])
 
 
 class SetupTest(unittest.TestCase):

From ec7eabbd3ba8dcf02d41afd7511507a089ae2026 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 22 May 2016 00:18:14 +0200
Subject: [PATCH 41/61] correct MANIFEST file

---
 MANIFEST | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST b/MANIFEST
index 11f635c3..8516e63d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -164,7 +164,6 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
-fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
@@ -182,6 +181,7 @@ fail2ban/server/filter.py
 fail2ban/server/filterpyinotify.py
 fail2ban/server/filtersystemd.py
 fail2ban/server/__init__.py
+fail2ban/server/ipdns.py
 fail2ban/server/iso8601.py
 fail2ban/server/jail.py
 fail2ban/server/jails.py

From 9bb869b8d4d92a3ffbba5e37ea74a0a8934971fa Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 21 May 2016 22:17:09 -0400
Subject: [PATCH 42/61] ENH: courier-smtp -- allow for trailing username (no
 spaces) in the logline

Closes #1440
---
 config/filter.d/courier-smtp.conf      | 2 +-
 fail2ban/tests/files/logs/courier-smtp | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf
index 7df385bf..fc0afc26 100644
--- a/config/filter.d/courier-smtp.conf
+++ b/config/filter.d/courier-smtp.conf
@@ -13,7 +13,7 @@ before = common.conf
 _daemon = courieresmtpd
 
 failregex = ^%(__prefix_line)serror,relay=<HOST>,.*: 550 User (<.*> )?unknown\.?$
-            ^%(__prefix_line)serror,relay=<HOST>,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?$
+            ^%(__prefix_line)serror,relay=<HOST>,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp
index 7beaf856..ab99d322 100644
--- a/fail2ban/tests/files/logs/courier-smtp
+++ b/fail2ban/tests/files/logs/courier-smtp
@@ -10,3 +10,5 @@ Jul  6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<a
 Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown.
 # failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" }
 Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=<firozquarl@aclunc.org>,to=<BOGUSUSER@HOSTEDDOMAIN.org>: 550 User unknown.
+# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" }
+Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin

From 9681cd3be8799c35ac1cf5dc69202b5bbd7ff001 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 22 May 2016 10:27:36 +0200
Subject: [PATCH 43/61] remove redundant IPAddr import

---
 fail2ban/server/actions.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 1b72af91..050c12e6 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -42,7 +42,6 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
-from .filter import IPAddr
 from .utils import Utils
 from ..helpers import getLogger
 

From 156065e70d3c71168c212a2c55a06e909f993b09 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 23 May 2016 15:33:45 +0200
Subject: [PATCH 44/61] splitwords: prevent to split to empty values by
 multiple separator characters together

---
 fail2ban/helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index fc51a88e..aef39835 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -136,4 +136,4 @@ def splitwords(s):
 	"""
 	if not s:
 		return []
-	return filter(bool, map(str.strip, re.split('[ ,\n]', s)))
+	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))

From a80043ce808cfb96ae5908c056ce8ccb969d9bc0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 25 May 2016 21:33:26 +0200
Subject: [PATCH 45/61] amend for gh-1419: tags substitution bug - wrong
 recognition of cyclic recursion, new test cases covered this

---
 fail2ban/server/action.py        | 25 ++++++++------
 fail2ban/tests/actiontestcase.py | 59 +++++++++++++++++++++++++++++---
 2 files changed, 69 insertions(+), 15 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index b1af659e..4901472d 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -55,6 +55,9 @@ _RETCODE_HINTS = {
 signame = dict((num, name)
 	for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))
 
+# max tag replacement count:
+MAX_TAG_REPLACE_COUNT = 10
+
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -390,26 +393,22 @@ class CommandAction(ActionBase):
 		"""
 		t = re.compile(r'<([^ <>]+)>')
 		# repeat substitution while embedded-recursive (repFlag is True)
+		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				if tag in cls._escapedTags:
-					# Escaped so won't match
-					continue
+				# ignore escaped or already done:
+				if tag in done: continue
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = {}
-				last_found = tag
+				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					curdone = done.get(last_found)
-					if curdone is None:
-						done[last_found] = curdone = []
-					if found_tag == tag or found_tag in curdone:
+					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						return False
@@ -421,8 +420,9 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag , tags[found_tag])
 					#logSys.log(5, 'value now: %s' % value)
-					curdone.append(found_tag)
-					last_found = found_tag
+					# increment reference count:
+					refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+					# the next match for replace:
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
@@ -431,6 +431,9 @@ class CommandAction(ActionBase):
 					if t.search(value):
 						repFlag = True
 					tags[tag] = value
+				# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
+				if '<' not in value: done.add(tag)
+			# stop interpolation, if no replacements anymore:
 			if not repFlag:
 				break
 		return tags
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index e20a6341..f33f3a5f 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -59,12 +59,62 @@ class CommandActionTest(LogCaptureTestCase):
 		# Unresolveable substition
 		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
-		# No-recursion, just multiple replacement of tag <T>, should be successful
-		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
-			self.assertEqual(CommandAction.substituteRecursiveTags(
-				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+		# We need here an ordered, because the sequence of iteration is very important for this test
+		if OrderedDict:
+			# No cyclic recursion, just multiple replacement of tag <T>, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+					(('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
 				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
 			)
+			# No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+				  (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'}
+			)
+			# No cyclic recursion, just multiple replacement of same tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((
+					('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'),
+					('ipmset', 'f2b-<name>'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables <lockingopt>'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '<multiport>'),
+					('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT',),
+				))
+				), OrderedDict((
+					('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('ipmset', 'f2b-any'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables -w'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT')
+				))
+			)
+			# Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle:
+			self.assertFalse(CommandAction.substituteRecursiveTags( OrderedDict((
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+					('DE', 'cycle <A>'),
+			)) ))
+			self.assertFalse(CommandAction.substituteRecursiveTags( OrderedDict((
+					('DE', 'cycle <A>'),
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+			)) ))
+			
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
@@ -132,6 +182,7 @@ class CommandActionTest(LogCaptureTestCase):
 				CallingMap(matches=lambda: str(10))),
 			"09 10 11")
 
+	def testReplaceNoTag(self):
 		# As tag not present, therefore callable should not be called
 		# Will raise ValueError if it is
 		self.assertEqual(

From 93f776d2eec1ae962642372e8f522dafc7eebb7d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 25 May 2016 21:33:26 +0200
Subject: [PATCH 46/61] amend for gh-1419: tags substitution bug - wrong
 recognition of cyclic recursion, new test cases covered this

---
 fail2ban/server/action.py        | 28 ++++++++-------
 fail2ban/tests/actiontestcase.py | 62 +++++++++++++++++++++++++++-----
 2 files changed, 69 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 51c5e077..f579c31e 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -46,9 +46,13 @@ _cmd_lock = threading.Lock()
 # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
 allowed_ipv6 = True
 
+# max tag replacement count:
+MAX_TAG_REPLACE_COUNT = 10
+
 # compiled RE for tag name (replacement name) 
 TAG_CRE = re.compile(r'<([^ <>]+)>')
 
+
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
 
@@ -356,32 +360,28 @@ class CommandAction(ActionBase):
 		tags = inptags.copy()
 		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
+		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				if tag in cls._escapedTags:
-					# Escaped so won't match
-					continue
+				# ignore escaped or already done:
+				if tag in done: continue
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = {}
-				last_found = tag
+				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					curdone = done.get(last_found)
-					if curdone is None:
-						done[last_found] = curdone = []
-					if found_tag == tag or found_tag in curdone:
+					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						raise ValueError(
 							"properties contain self referencing definitions "
 							"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-							(tag, found_tag, curdone, value))
+							(tag, found_tag, refCounts, value))
 					repl = None
 					if found_tag not in cls._escapedTags:
 						repl = tags.get(found_tag + '?' + conditional)
@@ -395,8 +395,9 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
-					curdone.append(found_tag)
-					last_found = found_tag
+					# increment reference count:
+					refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+					# the next match for replace:
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
@@ -405,6 +406,9 @@ class CommandAction(ActionBase):
 					if t.search(value):
 						repFlag = True
 					tags[tag] = value
+				# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
+				if '<' not in value: done.add(tag)
+			# stop interpolation, if no replacements anymore:
 			if not repFlag:
 				break
 		return tags
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index ac59a400..457d81bd 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -66,18 +66,62 @@ class CommandActionTest(LogCaptureTestCase):
 			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertRaises(ValueError,
 			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
-		# No-recursion, just multiple replacement of tag <T>, should be successful
-		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
-			self.assertEqual(CommandAction.substituteRecursiveTags(
-				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+		# We need here an ordered, because the sequence of iteration is very important for this test
+		if OrderedDict:
+			# No cyclic recursion, just multiple replacement of tag <T>, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+					(('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
 				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
 			)
-		# No-recursion, just multiple replacement of tag <T>, should be successful
-		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
-			self.assertEqual(CommandAction.substituteRecursiveTags(
-				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
-				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			# No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+				  (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'}
 			)
+			# No cyclic recursion, just multiple replacement of same tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((
+					('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'),
+					('ipmset', 'f2b-<name>'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables <lockingopt>'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '<multiport>'),
+					('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT',),
+				))
+				), OrderedDict((
+					('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('ipmset', 'f2b-any'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables -w'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT')
+				))
+			)
+			# Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle:
+			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+					('DE', 'cycle <A>'),
+			)) ))
+			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+					('DE', 'cycle <A>'),
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+			)) ))
+			
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})

From b7787f4af496c9b57814f77fa21873dfa14cfb62 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 30 May 2016 12:30:51 +0200
Subject: [PATCH 47/61] use raw host (don't use textToIp) if usedns not yes or
 warn

---
 fail2ban/server/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index d561b507..1e751057 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -523,7 +523,7 @@ class Filter(JailThread):
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
 					try:
 						host = failRegex.getHost()
-						if returnRawHost:
+						if returnRawHost or self.__useDns not in ("yes", "warn"):
 							failList.append([failRegexIndex, host, date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:

From c33e25bab6a370a16ce5e16fdb8c4e564c06b92f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 30 May 2016 13:32:58 +0200
Subject: [PATCH 48/61] fail2ban-regex: extended with option '--raw' (switch to
 raw host, prevent resolving of ip)

---
 fail2ban/client/fail2banregex.py        | 5 ++++-
 fail2ban/tests/fail2banregextestcase.py | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 7c56a77e..c2b2e894 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -126,6 +126,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   help="set custom pattern used to match date/times"),
 		Option("-e", "--encoding",
 			   help="File encoding. Default: system locale"),
+		Option("-r", "--raw", action='store_true',
+			   help="Raw hosts, don't resolve dns"),
 		Option("-L", "--maxlines", type=int, default=0,
 			   help="maxlines for multi-line regex"),
 		Option("-m", "--journalmatch",
@@ -239,6 +241,7 @@ class Fail2banRegex(object):
 			self.encoding = opts.encoding
 		else:
 			self.encoding = locale.getpreferredencoding()
+		self.raw = True if opts.raw else False
 
 	def decode_line(self, line):
 		return FileContainer.decode_line('<LOG>', self.encoding, line)
@@ -335,7 +338,7 @@ class Fail2banRegex(object):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
-			line, ret = self._filter.processLine(line, date, checkAllRegex=True)
+			line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw)
 			for match in ret:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 49d6a3a6..3321ffd8 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -132,6 +132,15 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('Dez 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128')
 		self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10')
 
+	def testDirectRE_1raw(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--print-all-matched", "--raw",
+			Fail2banRegexTest.FILENAME_01, 
+			Fail2banRegexTest.RE_00
+		)
+		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
+
 	def testDirectRE_2(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"--print-all-matched",

From 8ec4e1189e0eb904473c1b7fcaa99e800e93911b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 30 May 2016 15:34:21 +0200
Subject: [PATCH 49/61] use raw host (don't use textToIp) if usedns exactly
 `raw`, because `usedns = no` should ignore no ip failures

---
 config/jail.conf          | 1 +
 fail2ban/server/filter.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index cea9a06e..941737ff 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -94,6 +94,7 @@ backend = auto
 #        but it will be logged as a warning.
 # no:    if a hostname is encountered, will not be used for banning,
 #        but it will be logged as info.
+# raw:   use raw value (no hostname), allow use it for no-host filters/actions (example user)
 usedns = warn
 
 # "logencoding" specifies the encoding of the log files handled by the jail
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 1e751057..3901fae9 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -523,7 +523,7 @@ class Filter(JailThread):
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
 					try:
 						host = failRegex.getHost()
-						if returnRawHost or self.__useDns not in ("yes", "warn"):
+						if returnRawHost or self.__useDns == "raw":
 							failList.append([failRegexIndex, host, date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:

From 48a8324662afdeed4016fd4630712331af66d610 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 30 May 2016 11:02:12 -0400
Subject: [PATCH 50/61] ENH: use non-capturing regex groups in exim-common and
 exim filters

---
 config/filter.d/exim-common.conf |  4 ++--
 config/filter.d/exim.conf        | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf
index 8f9553fe..0e1b74fa 100644
--- a/config/filter.d/exim-common.conf
+++ b/config/filter.d/exim-common.conf
@@ -9,8 +9,8 @@ after = exim-common.local
 
 [Definition]
 
-host_info = (H=([\w.-]+ )?(\(\S+\) )?)?\[<HOST>\](:\d+)? (I=\[\S+\](:\d+)? )?(U=\S+ )?(P=e?smtp )?
-pid = ( \[\d+\])?
+host_info = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?\[<HOST>\](?::\d+)? (?:I=\[\S+\](:\d+)? )?(?:U=\S+ )?(?:P=e?smtp )?
+pid = (?: \[\d+\])?
 
 # DEV Notes:
 # From exim source code: ./src/receive.c:add_host_info_for_log
diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 517e3de7..1ef74b01 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -14,13 +14,13 @@ before = exim-common.conf
 [Definition]
 
 failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
-            ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[<HOST>\](:\d+)?( I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$
-            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
+            ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[<HOST>\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+            ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$
+            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(| \S*)" %(host_info)sAUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (|\S* )(|\(\S*\) )%(host_info)sD=\d+s(| C=\S*)\s*$
-            ^%(pid)s \S+ SMTP connection from (|\S* )(|\(\S*\) )%(host_info)sclosed by DROP in ACL\s*$
+            ^%(pid)s SMTP protocol error in "AUTH \S*(?:| \S*)" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s no MAIL in SMTP connection from (?:|\S* )(?:|\(\S*\) )%(host_info)sD=\d+s(?:| C=\S*)\s*$
+            ^%(pid)s \S+ SMTP connection from (?:|\S* )(?:|\(\S*\) )%(host_info)sclosed by DROP in ACL\s*$
 
 ignoreregex = 
 

From 64346614804e91285bfb1fcbb407d0e1cdff618e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 30 May 2016 12:12:53 -0400
Subject: [PATCH 51/61] RF: for consistency use (?:XXX)? instead of (?:|XXX)

---
 config/filter.d/exim.conf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 1ef74b01..a1d699c0 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -18,9 +18,9 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(?:| \S*)" %(host_info)sAUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (?:|\S* )(?:|\(\S*\) )%(host_info)sD=\d+s(?:| C=\S*)\s*$
-            ^%(pid)s \S+ SMTP connection from (?:|\S* )(?:|\(\S*\) )%(host_info)sclosed by DROP in ACL\s*$
+            ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s no MAIL in SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$
+            ^%(pid)s \S+ SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 
 ignoreregex = 
 

From b3bb8b20bfe92d1391af510635cd4200d7089622 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 31 May 2016 17:45:44 +0200
Subject: [PATCH 52/61] amend for new option of `usedns=raw` - forgotten
 validation fix inside setUseDns

---
 fail2ban/server/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 3901fae9..45ae7704 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -168,7 +168,7 @@ class Filter(JailThread):
 		if isinstance(value, bool):
 			value = {True: 'yes', False: 'no'}[value]
 		value = value.lower()			  # must be a string by now
-		if not (value in ('yes', 'no', 'warn')):
+		if not (value in ('yes', 'warn', 'no', 'raw')):
 			logSys.error("Incorrect value %r specified for usedns. "
 						 "Using safe 'no'" % (value,))
 			value = 'no'

From d3442742716e8253ca592831ae61b51305571fb1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 31 May 2016 21:55:26 +0200
Subject: [PATCH 53/61] separated host match group and tags for ip4, ip6, dns,
 fid (failure-id): - better recognition for usage of textToIp, expected or raw
 value should be used; - separated failure id vs. host (if found use `fid`
 instead of `host` resp. `ip`); - additional optional groups may be used in
 tags replacement by executing actions;

---
 fail2ban/server/failmanager.py    | 26 ++++-----
 fail2ban/server/failregex.py      | 95 +++++++++++++++++++++++-------
 fail2ban/server/filter.py         | 50 +++++++++++++---
 fail2ban/server/ipdns.py          | 96 ++++++++++++++++---------------
 fail2ban/server/ticket.py         | 10 +++-
 fail2ban/tests/samplestestcase.py |  2 +-
 fail2ban/tests/servertestcase.py  | 50 ++++++++++++++++
 7 files changed, 237 insertions(+), 92 deletions(-)

diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index b342b280..ee4b049d 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -78,9 +78,9 @@ class FailManager:
 	def addFailure(self, ticket, count=1):
 		attempts = 1
 		with self.__lock:
-			ip = ticket.getIP()
+			fid = ticket.getID()
 			try:
-				fData = self.__failList[ip]
+				fData = self.__failList[fid]
 				# if the same object - the same matches but +1 attempt:
 				if fData is ticket:
 					matches = None
@@ -109,7 +109,7 @@ class FailManager:
 					fData = FailTicket(ticket=ticket)
 				if count > ticket.getAttempt():
 					fData.setRetry(count)
-				self.__failList[ip] = fData
+				self.__failList[fid] = fData
 
 			attempts = fData.getRetry()
 			self.__failTotal += 1
@@ -132,7 +132,7 @@ class FailManager:
 	
 	def cleanup(self, time):
 		with self.__lock:
-			todelete = [ip for ip,item in self.__failList.iteritems() \
+			todelete = [fid for fid,item in self.__failList.iteritems() \
 				if item.getLastTime() + self.__maxTime <= time]
 			if len(todelete) == len(self.__failList):
 				# remove all:
@@ -142,27 +142,27 @@ class FailManager:
 				return
 			if len(todelete) / 2.0 <= len(self.__failList) / 3.0:
 				# few as 2/3 should be removed - remove particular items:
-				for ip in todelete:
-					del self.__failList[ip]
+				for fid in todelete:
+					del self.__failList[fid]
 			else:
 				# create new dictionary without items to be deleted:
-				self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \
+				self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \
 					if item.getLastTime() + self.__maxTime > time)
 		self.__bgSvc.service()
 	
-	def delFailure(self, ip):
+	def delFailure(self, fid):
 		with self.__lock:
 			try:
-				del self.__failList[ip]
+				del self.__failList[fid]
 			except KeyError:
 				pass
 	
-	def toBan(self, ip=None):
+	def toBan(self, fid=None):
 		with self.__lock:
-			for ip in ([ip] if ip != None and ip in self.__failList else self.__failList):
-				data = self.__failList[ip]
+			for fid in ([fid] if fid != None and fid in self.__failList else self.__failList):
+				data = self.__failList[fid]
 				if data.getRetry() >= self.__maxRetry:
-					del self.__failList[ip]
+					del self.__failList[fid]
 					return data
 		self.__bgSvc.service()
 		raise FailManagerEmpty
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 6076acc3..d343dbe0 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -62,18 +62,39 @@ class Regex:
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
 
+	##
+	# Replaces "<HOST>", "<IP4>", "<IP6>", "<FID>" with default regular expression for host
+	#
+	# (see gh-1374 for the discussion about other candidates)
+	# @return the replaced regular expression as string
+
 	@staticmethod
 	def _resolveHostTag(regex):
-		# Replace "<HOST>" with default regular expression for host:
-		# Other candidates (see gh-1374 for the discussion about):
-		#   differentiate: r"""(?:(?:::f{4,6}:)?(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<HOST>[\w\-.^_]*\w))"""
-		#     expected many changes in filter, failregex, etc...
-		#   simple:        r"""(?:::f{4,6}:)?(?P<host>[\w\-.^_:]*\w)"""
-		#     not good enough, if not precise expressions around <HOST>, because for example will match '1.2.3.4:23930' as ip-address;
-		# Todo: move this functionality to filter reader, as default <HOST> replacement,
-		#       make it configurable (via jail/filter configs)
-		return regex.replace("<HOST>",
-		 	r"""(?:::f{4,6}:)?(?P<host>(?:\d{1,3}\.){3}\d{1,3}|\[?(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}\]?|(?<=:):)|[\w\-.^_]*\w)""")
+		# 3 groups instead of <HOST> - separated ipv4, ipv6 and host
+		regex = regex.replace("<HOST>",
+		 	r"""(?:(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<dns>[\w\-.^_]*\w))""")
+		# separated ipv4:
+		r = r"""(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})"""
+		regex = regex.replace("<IP4>", r); # self closed
+		regex = regex.replace("<F-IP4/>", r); # closed
+		# separated ipv6:
+		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}?|(?<=:):))"""
+		regex = regex.replace("<IP6>", r); # self closed
+		regex = regex.replace("<F-IP6/>", r); # closed
+		# separated dns:
+		r = r"""(?P<dns>[\w\-.^_]*\w)"""
+		regex = regex.replace("<DNS>", r); # self closed
+		regex = regex.replace("<F-DNS/>", r); # closed
+		# default failure-id as no space tag:
+		regex = regex.replace("<F-ID/>", r"""(?P<fid>\S+)"""); # closed
+		# default failure port, like 80 or http :
+		regex = regex.replace("<F-PORT/>", r"""(?P<port>\w+)"""); # closed
+		# default failure groups (begin / end tag) for customizable expressions:
+		for o,r in (('IP4', 'ip4'), ('IP6', 'ip6'), ('DNS', 'dns'), ('ID', 'fid'), ('PORT', 'fport')):
+			regex = regex.replace("<F-%s>" % o, "(?P<%s>" % r); # open tag
+			regex = regex.replace("</F-%s>" % o, ")"); # close tag
+
+		return regex
 
 	##
 	# Gets the regular expression.
@@ -207,6 +228,13 @@ class RegexException(Exception):
 	pass
 
 
+##
+# Groups used as failure identifier.
+#
+# The order of this tuple is important while searching for failure-id
+#
+FAILURE_ID_GROPS = ("fid", "ip4", "ip6", "dns")
+
 ##
 # Regular expression class.
 #
@@ -220,25 +248,48 @@ class FailRegex(Regex):
 	# Creates a new object. This method can throw RegexException in order to
 	# avoid construction of invalid object.
 	# @param value the regular expression
-	
+
 	def __init__(self, regex):
 		# Initializes the parent.
 		Regex.__init__(self, regex)
-		# Check for group "host"
-		if "host" not in self._regexObj.groupindex:
-			raise RegexException("No 'host' group in '%s'" % self._regex)
+		# Check for group "dns", "ip4", "ip6", "fid"
+		if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]:
+			raise RegexException("No failure-id group in '%s'" % self._regex)
 	
 	##
-	# Returns the matched host.
+	# Returns all matched groups.
 	#
-	# This corresponds to the pattern matched by the named group "host".
-	# @return the matched host
+
+	def getGroups(self):
+		return self._matchCache.groupdict()
+
+	##
+	# Returns the matched failure id.
+	#
+	# This corresponds to the pattern matched by the named group from given groups.
+	# @return the matched failure-id
 	
-	def getHost(self):
-		host = self._matchCache.group("host")
-		if host is None:
+	def getFailID(self, groups=FAILURE_ID_GROPS):
+		fid = None
+		for grp in groups:
+			try:
+				fid = self._matchCache.group(grp)
+			except IndexError:
+				continue
+			if fid is not None:
+				break
+		if fid is None:
 			# Gets a few information.
 			s = self._matchCache.string
 			r = self._matchCache.re
-			raise RegexException("No 'host' found in '%s' using '%s'" % (s, r))
-		return str(host)
+			raise RegexException("No group found in '%s' using '%s'" % (s, r))
+		return str(fid)
+
+	##
+	# Returns the matched host.
+	#
+	# This corresponds to the pattern matched by the named group "ip4", "ip6" or "dns".
+	# @return the matched host
+	
+	def getHost(self):
+		return self.getFailID(("ip4", "ip6", "dns"))
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c1cd2d09..d0d8f74e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -418,6 +418,9 @@ class Filter(JailThread):
 			ip = element[1]
 			unixTime = element[2]
 			lines = element[3]
+			fail = {}
+			if len(element) > 4:
+				fail = element[4]
 			logSys.debug("Processing line with time:%s and ip:%s", 
 					unixTime, ip)
 			if unixTime < MyTime.time() - self.getFindTime():
@@ -429,7 +432,7 @@ class Filter(JailThread):
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 			)
-			tick = FailTicket(ip, unixTime, lines)
+			tick = FailTicket(ip, unixTime, lines, data=fail)
 			self.failManager.addFailure(tick)
 
 	##
@@ -457,7 +460,12 @@ class Filter(JailThread):
 		checkAllRegex=False):
 		failList = list()
 
-		# Checks if we must ignore this line.
+		cidr = IPAddr.CIDR_UNSPEC
+		if self.__useDns == "raw":
+			returnRawHost = True
+			cidr = IPAddr.CIDR_RAW
+
+		# Checks if we mut ignore this line.
 		if self.ignoreLine([tupleLine[::2]]) is not None:
 			# The ignoreregex matched. Return.
 			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
@@ -518,19 +526,45 @@ class Filter(JailThread):
 						 % ("\n".join(failRegex.getMatchedLines()), timeText))
 				else:
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+					# retrieve failure-id, host, etc from failure match:
+					raw = returnRawHost
 					try:
-						host = failRegex.getHost()
-						if returnRawHost or self.__useDns == "raw":
-							failList.append([failRegexIndex, IPAddr(host), date,
-								 failRegex.getMatchedLines()])
+						fail = failRegex.getGroups()
+						# failure-id:
+						fid = fail.get('fid')
+						# ip-address or host:
+						host = fail.get('ip4')
+						if host is not None:
+							raw = True
+						else:
+							host = fail.get('ip6')
+							if host is not None:
+								raw = True
+							else:
+								host = fail.get('dns')
+								if host is None:
+									# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
+									if fid is None:
+										fid = failRegex.getFailID()
+									host = fid
+									cidr = IPAddr.CIDR_RAW
+						# if raw - add single ip or failure-id,
+						# otherwise expand host to multiple ips using dns (or ignore it if not valid):
+						if raw:
+							ip = IPAddr(host, cidr)
+							# check host equal failure-id, if not - failure with complex id:
+							if fid is not None and fid != host:
+								ip = IPAddr(fid, IPAddr.CIDR_RAW)
+							failList.append([failRegexIndex, ip, date,
+								failRegex.getMatchedLines(), fail])
 							if not checkAllRegex:
 								break
 						else:
 							ips = DNSUtils.textToIp(host, self.__useDns)
 							if ips:
 								for ip in ips:
-									failList.append([failRegexIndex, ip, 
-										 date, failRegex.getMatchedLines()])
+									failList.append([failRegexIndex, ip, date,
+										failRegex.getMatchedLines(), fail])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 8109c21a..61c1ba68 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -138,18 +138,21 @@ class IPAddr(object):
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
 
-	def __new__(cls, ipstr, cidr=-1):
+	CIDR_RAW = -2
+	CIDR_UNSPEC = -1
+
+	def __new__(cls, ipstr, cidr=CIDR_UNSPEC):
 		# check already cached as IPAddr
 		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
 		if ip is not None:
 			return ip
 		# wrap mask to cidr (correct plen):
-		if cidr == -1:
+		if cidr == IPAddr.CIDR_UNSPEC:
 			ipstr, cidr = IPAddr.__wrap_ipstr(ipstr)
 			args = (ipstr, cidr)
 			# check cache again:
-			if cidr != -1:
+			if cidr != IPAddr.CIDR_UNSPEC:
 				ip = IPAddr.CACHE_OBJ.get(args)
 				if ip is not None:
 					return ip
@@ -166,7 +169,7 @@ class IPAddr(object):
 			ipstr = ipstr[1:-1]
 		# test mask:
 		if "/" not in ipstr:
-			return ipstr, -1
+			return ipstr, IPAddr.CIDR_UNSPEC
 		s = ipstr.split('/', 1)
 		# IP address without CIDR mask
 		if len(s) > 2:
@@ -176,7 +179,7 @@ class IPAddr(object):
 		s[1] = long(s[1])
 		return s
 		
-	def __init(self, ipstr, cidr=-1):
+	def __init(self, ipstr, cidr=CIDR_UNSPEC):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -184,49 +187,48 @@ class IPAddr(object):
 		self._addr = 0
 		self._plen = 0
 		self._maskplen = None
-		self._raw = ""
+		# always save raw value (normally used if really raw or not valid only):
+		self._raw = ipstr
+		# if not raw - recognize family, set addr, etc.:
+		if cidr != IPAddr.CIDR_RAW:
+			for family in [socket.AF_INET, socket.AF_INET6]:
+				try:
+					binary = socket.inet_pton(family, ipstr)
+					self._family = family
+					break
+				except socket.error:
+					continue
 
-		for family in [socket.AF_INET, socket.AF_INET6]:
-			try:
-				binary = socket.inet_pton(family, ipstr)
-				self._family = family
-				break
-			except socket.error:
-				continue
-
-		if self._family == socket.AF_INET:
-			# convert host to network byte order
-			self._addr, = struct.unpack("!L", binary)
-			self._plen = 32
-
-			# mask out host portion if prefix length is supplied
-			if cidr is not None and cidr >= 0:
-				mask = ~(0xFFFFFFFFL >> cidr)
-				self._addr &= mask
-				self._plen = cidr
-
-		elif self._family == socket.AF_INET6:
-			# convert host to network byte order
-			hi, lo = struct.unpack("!QQ", binary)
-			self._addr = (hi << 64) | lo
-			self._plen = 128
-
-			# mask out host portion if prefix length is supplied
-			if cidr is not None and cidr >= 0:
-				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self._addr &= mask
-				self._plen = cidr
-
-			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr.IP6_4COMPAT):
-				self._addr = lo & 0xFFFFFFFFL
-				self._family = socket.AF_INET
+			if self._family == socket.AF_INET:
+				# convert host to network byte order
+				self._addr, = struct.unpack("!L", binary)
 				self._plen = 32
+
+				# mask out host portion if prefix length is supplied
+				if cidr is not None and cidr >= 0:
+					mask = ~(0xFFFFFFFFL >> cidr)
+					self._addr &= mask
+					self._plen = cidr
+
+			elif self._family == socket.AF_INET6:
+				# convert host to network byte order
+				hi, lo = struct.unpack("!QQ", binary)
+				self._addr = (hi << 64) | lo
+				self._plen = 128
+
+				# mask out host portion if prefix length is supplied
+				if cidr is not None and cidr >= 0:
+					mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+					self._addr &= mask
+					self._plen = cidr
+
+				# if IPv6 address is a IPv4-compatible, make instance a IPv4
+				elif self.isInNet(IPAddr.IP6_4COMPAT):
+					self._addr = lo & 0xFFFFFFFFL
+					self._family = socket.AF_INET
+					self._plen = 32
 		else:
-			# string couldn't be converted neither to a IPv4 nor
-			# to a IPv6 address - retain raw input for later use
-			# (e.g. DNS resolution)
-			self._raw = ipstr
+			self._family = IPAddr.CIDR_RAW
 
 	def __repr__(self):
 		return self.ntoa
@@ -270,6 +272,8 @@ class IPAddr(object):
 		return self._family != socket.AF_UNSPEC
 
 	def __eq__(self, other):
+		if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr):
+			return self._raw == other
 		if not isinstance(other, IPAddr):
 			if other is None: return False
 			other = IPAddr(other)
@@ -285,6 +289,8 @@ class IPAddr(object):
 		return not (self == other)
 
 	def __lt__(self, other):
+		if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr):
+			return self._raw < other
 		if not isinstance(other, IPAddr):
 			if other is None: return False
 			other = IPAddr(other)
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 130de4f2..65ed83c3 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -36,7 +36,7 @@ logSys = getLogger(__name__)
 
 class Ticket:
 	
-	def __init__(self, ip=None, time=None, matches=None, ticket=None):
+	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		"""Ticket constructor
 
 		@param ip the IP address
@@ -50,6 +50,7 @@ class Ticket:
 		self._banTime = None;
 		self._time = time if time is not None else MyTime.time()
 		self._data = {'matches': [], 'failures': 0}
+		self._data.update(data)
 		if ticket:
 			# ticket available - copy whole information from ticket:
 			self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__)
@@ -78,6 +79,9 @@ class Ticket:
 			value = IPAddr(value)
 		self.__ip = value
 	
+	def getID(self):
+		return self._data.get('fid', self.__ip)
+	
 	def getIP(self):
 		return self.__ip
 	
@@ -164,12 +168,12 @@ class Ticket:
 
 class FailTicket(Ticket):
 
-	def __init__(self, ip=None, time=None, matches=None, ticket=None):
+	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		# this class variables:
 		self.__retry = 0
 		self.__lastReset = None
 		# create/copy using default ticket constructor:
-		Ticket.__init__(self, ip, time, matches, ticket)
+		Ticket.__init__(self, ip, time, matches, data, ticket)
 		# init:
 		if ticket is None:
 			self.__lastReset = time if time is not None else self.getTime()
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 326d09f7..7af12f13 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -127,7 +127,7 @@ def testSampleRegexsFactory(name, basedir):
 								 (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
 
 				# Verify timestamp and host as expected
-				failregex, host, fail2banTime, lines = ret[0]
+				failregex, host, fail2banTime, lines, fail = ret[0]
 				self.assertEqual(host, faildata.get("host", None))
 
 				t = faildata.get("time", None)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 7557a501..9eb88c21 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -933,6 +933,14 @@ class RegexTests(unittest.TestCase):
 
 	def testHost(self):
 		self.assertRaises(RegexException, FailRegex, '')
+		self.assertRaises(RegexException, FailRegex, '^test no group$')
+		self.assertTrue(FailRegex('^test <HOST> group$'))
+		self.assertTrue(FailRegex('^test <IP4> group$'))
+		self.assertTrue(FailRegex('^test <IP6> group$'))
+		self.assertTrue(FailRegex('^test <DNS> group$'))
+		self.assertTrue(FailRegex('^test id group: ip:port = <F-ID><IP4>(?::<F-PORT/>)?</F-ID>$'))
+		self.assertTrue(FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$'))
+		self.assertTrue(FailRegex('^test id group: anything = <F-ID/>$'))
 		# Testing obscure case when host group might be missing in the matched pattern,
 		# e.g. if we made it optional.
 		fr = FailRegex('%%<HOST>?')
@@ -940,6 +948,30 @@ class RegexTests(unittest.TestCase):
 		fr.search([('%%',"","")])
 		self.assertTrue(fr.hasMatched())
 		self.assertRaises(RegexException, fr.getHost)
+		# The same as above but using separated IPv4/IPv6 expressions
+		fr = FailRegex('%%inet(?:=<F-IP4/>|inet6=<F-IP6/>)?')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('%%inet=test',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertRaises(RegexException, fr.getHost)
+		# Success case: using separated IPv4/IPv6 expressions (no HOST)
+		fr = FailRegex('%%(?:inet(?:=<IP4>|6=<IP6>)?|dns=<DNS>?)')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('%%inet=192.0.2.1',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), '192.0.2.1')
+		fr.search([('%%inet6=2001:DB8::',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), '2001:DB8::')
+		fr.search([('%%dns=example.com',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), 'example.com')
+		# Success case: using user as failure-id
+		fr = FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('test id group: user:(test login name)',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getFailID(), 'test login name')
 
 
 class _BadThread(JailThread):
@@ -998,6 +1030,24 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
 		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
 
+	def test_IPAddr_Raw(self):
+		# raw string:
+		r = IPAddr('xxx', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, 'xxx')
+		self.assertEqual('xxx', str(r))
+		self.assertNotEqual(r, IPAddr('xxx'))
+		# raw (not IP, for example host:port as string):
+		r = IPAddr('1:2', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, '1:2')
+		self.assertEqual('1:2', str(r))
+		self.assertNotEqual(r, IPAddr('1:2'))
+
 	def _testExecActions(self, server):
 		jails = server._Server__jails
 		for jail in jails:

From ced6c8307b66e1959a8fd4262b561229cb9fef60 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 2 Jun 2016 20:56:28 -0400
Subject: [PATCH 54/61] BF: finalize that sample log line for exim4

was intended in 743a531eb59a4f3704dda68557fb34481538e159 to be an entry without a port
after the [host]
---
 fail2ban/tests/files/logs/exim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index 9da5ff22..9053bf8d 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -55,7 +55,7 @@
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
 2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=10s
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
-2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=10s
+2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116] I=[172.89.0.6]:25 D=10s
 # failJSON: { "time": "2016-03-21T04:07:49", "match": true , "host": "174.137.147.204" }
 2016-03-21 04:07:49 [25874] 1ahr79-0006jK-G9 SMTP connection from (voyeur.webair.com) [174.137.147.204]:44884 I=[172.89.0.6]:25 closed by DROP in ACL
 # failJSON: { "time": "2016-03-21T04:33:13", "match": true , "host": "206.214.71.53" }

From f85fb45b29768f687546ba25f805977cf00b6e43 Mon Sep 17 00:00:00 2001
From: Ludovic Gasc <gmludo@gmail.com>
Date: Tue, 7 Jun 2016 11:40:35 +0200
Subject: [PATCH 55/61] Asterisk pjsip (#1456)

* Improve PJSIP log support for Asterisk 13+

* Update changelog: filter.d/asterisk.conf - fix security log support for PJSIP and Asterisk 13+

* Change pjsip regexp with sebres observation, thanks to @nturcksin
---
 ChangeLog                          | 1 +
 config/filter.d/asterisk.conf      | 1 +
 fail2ban/tests/files/logs/asterisk | 4 ++++
 3 files changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 76719f16..21b8adfc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -25,6 +25,7 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
        added new parameter `__date_ambit`
    * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, 
      not argument of fail2ban (see gh-1434)
+   * filter.d/asterisk.conf - fix security log support for PJSIP and Asterisk 13+
 
 - New Features:
    * New Actions:
diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 01063efa..f6ccdd4f 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -27,6 +27,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
+            ^%(__prefix_line)s%(log_prefix)s Request from '[^']*' failed for '<HOST>(?::\d+)?' \(callid: \w*\) - No matching endpoint found$
 
 ignoreregex =
 
diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index aa32a290..3f49beec 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -67,3 +67,7 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 [2016-01-28 10:34:31] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0+441772285407' rejected because extension not found in context 'default'.
 # failJSON: { "time": "2016-01-28T10:34:33", "match": true , "host": "1.2.3.4" }
 [2016-01-28 10:34:33] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '' rejected because extension not found in context 'my-context'.
+
+# Failed authentication with pjsip on Asterisk 13+
+# failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
+[2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - No matching endpoint found
\ No newline at end of file

From 11f7cf5ad82b26c59f5d62c35b38343264483980 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 7 Jun 2016 21:38:39 -0400
Subject: [PATCH 56/61] DOC: changelog for recent exim filters tune up

---
 ChangeLog | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 21b8adfc..3f985b20 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,6 +30,7 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
 - New Features:
    * New Actions:
      - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
+
 - Enhancements:
    * Extreme speedup of all sqlite database operations (gh-1436),
      by using of following sqlite options:
@@ -38,7 +39,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
      - (temp_store = MEMORY) temporary tables and indices are kept in memory
    * journald journalmatch for pure-ftpd (gh-1362)
    * Add additional regex filter for dovecot ldap authentication failures (gh-1370)
-   * added additional regex filters for exim (gh-1371)
+   * filter.d/exim*conf
+     - added additional regexes (gh-1371)
+     - made port entry optional
 
 
 ver. 0.9.4 (2016/03/08) - for-you-ladies

From e40a8c8ae832504107d2ebdd69903e516c89ca7f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 11:22:14 +0200
Subject: [PATCH 57/61] small code review

---
 fail2ban/server/filter.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index d0d8f74e..318d8e49 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -533,21 +533,17 @@ class Filter(JailThread):
 						# failure-id:
 						fid = fail.get('fid')
 						# ip-address or host:
-						host = fail.get('ip4')
+						host = fail.get('ip4') or fail.get('ip6')
 						if host is not None:
 							raw = True
 						else:
-							host = fail.get('ip6')
-							if host is not None:
-								raw = True
-							else:
-								host = fail.get('dns')
-								if host is None:
-									# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
-									if fid is None:
-										fid = failRegex.getFailID()
-									host = fid
-									cidr = IPAddr.CIDR_RAW
+							host = fail.get('dns')
+							if host is None:
+								# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
+								if fid is None:
+									fid = failRegex.getFailID()
+								host = fid
+								cidr = IPAddr.CIDR_RAW
 						# if raw - add single ip or failure-id,
 						# otherwise expand host to multiple ips using dns (or ignore it if not valid):
 						if raw:

From 8893473d82f3732cfec6c7e302e1b94ff6b62b99 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 11:47:24 +0200
Subject: [PATCH 58/61] pypy fix: KeyError instead of IndexError by missing
 group

---
 fail2ban/server/failregex.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index d343dbe0..12b8986a 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -274,7 +274,7 @@ class FailRegex(Regex):
 		for grp in groups:
 			try:
 				fid = self._matchCache.group(grp)
-			except IndexError:
+			except (IndexError, KeyError):
 				continue
 			if fid is not None:
 				break

From e39126f6301fdf86348f43283b802dc28ad2a76f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 13:15:46 +0200
Subject: [PATCH 59/61] badip timeout option introduced, set to 30 seconds in
 our test cases

---
 config/action.d/badips.py              | 11 +++++++----
 fail2ban/tests/action_d/test_badips.py |  1 +
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/config/action.d/badips.py b/config/action.d/badips.py
index 025289ca..4bc879a1 100644
--- a/config/action.d/badips.py
+++ b/config/action.d/badips.py
@@ -80,14 +80,17 @@ class BadIPsAction(ActionBase):
 		If invalid `category`, `score`, `banaction` or `updateperiod`.
 	"""
 
+	TIMEOUT = 10
 	_badips = "http://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"):
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
+		self.timeout = timeout
 		self.agent = agent
 		self.category = category
 		self.score = score
@@ -119,7 +122,7 @@ class BadIPsAction(ActionBase):
 		"""
 		try:
 			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), None, 3)
+				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -173,7 +176,7 @@ class BadIPsAction(ActionBase):
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -358,7 +361,7 @@ class BadIPsAction(ActionBase):
 			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 74594420..97dabada 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -39,6 +39,7 @@ if sys.version_info >= (2,7):
 			self.jail.actions.add("badips", pythonModule, initOpts={
 				'category': "ssh",
 				'banaction': "test",
+				'timeout': (3 if unittest.F2B.fast else 30),
 				})
 			self.action = self.jail.actions["badips"]
 

From 2efcf3c17be9414a8fdbb9f748b38a1b260bd7be Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 13:17:18 +0200
Subject: [PATCH 60/61] increase of test coverage (+ test_IPAddr moved to one
 place), small bugs fixed

---
 fail2ban/server/ipdns.py         |  7 +--
 fail2ban/tests/filtertestcase.py | 84 ++++++++++++++++++++++++++++++++
 fail2ban/tests/servertestcase.py | 22 ---------
 3 files changed, 86 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 61c1ba68..cca63535 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -85,10 +85,7 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			if not isinstance(ip, IPAddr):
-				v = socket.gethostbyaddr(ip)[0]
-			else:
-				v = socket.gethostbyaddr(ip.ntoa)[0]
+			v = socket.gethostbyaddr(ip)[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -359,7 +356,7 @@ class IPAddr(object):
 			if not suffix:
 				suffix = "in-addr.arpa."
 		elif self.isIPv6:
-			exploded_ip = self.hexdump()
+			exploded_ip = self.hexdump
 			if not suffix:
 				suffix = "ip6.arpa."
 		else:
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 4f5dbb55..dd3468ed 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1353,6 +1353,42 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
 
+	def test_IPAddr(self):
+		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
+		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
+
+	def test_IPAddr_Raw(self):
+		# raw string:
+		r = IPAddr('xxx', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, 'xxx')
+		self.assertEqual('xxx', str(r))
+		self.assertNotEqual(r, IPAddr('xxx'))
+		# raw (not IP, for example host:port as string):
+		r = IPAddr('1:2', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, '1:2')
+		self.assertEqual('1:2', str(r))
+		self.assertNotEqual(r, IPAddr('1:2'))
+		# raw vs ip4 (raw is not an ip):
+		r = IPAddr('93.184.0.1', IPAddr.CIDR_RAW)
+		ip4 = IPAddr('93.184.0.1')
+		self.assertNotEqual(ip4, r)
+		self.assertNotEqual(r, ip4)
+		self.assertTrue(r < ip4)
+		self.assertTrue(r < ip4)
+		# raw vs ip6 (raw is not an ip):
+		r = IPAddr('1::2', IPAddr.CIDR_RAW)
+		ip6 = IPAddr('1::2')
+		self.assertNotEqual(ip6, r)
+		self.assertNotEqual(r, ip6)
+		self.assertTrue(r < ip6)
+		self.assertTrue(r < ip6)
+
 	def testUseDns(self):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
@@ -1377,14 +1413,25 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 				self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
+		# pure ips:
+		for s in ('93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'):
+			ips = DNSUtils.textToIp(s, 'yes')
+			self.assertEqual(ips, [s])
+			self.assertTrue(isinstance(ips[0], IPAddr))
 
 	def testIpToName(self):
 		unittest.F2B.SkipIfNoNetwork()
 		res = DNSUtils.ipToName('8.8.4.4')
 		self.assertEqual(res, 'google-public-dns-b.google.com')
+		# same as above, but with IPAddr:
+		res = DNSUtils.ipToName(IPAddr('8.8.4.4'))
+		self.assertEqual(res, 'google-public-dns-b.google.com')
 		# invalid ip (TEST-NET-1 according to RFC 5737)
 		res = DNSUtils.ipToName('192.0.2.0')
 		self.assertEqual(res, None)
+		# invalid ip:
+		res = DNSUtils.ipToName('192.0.2.888')
+		self.assertEqual(res, None)
 
 	def testAddr2bin(self):
 		res = IPAddr('10.0.0.0')
@@ -1398,11 +1445,44 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = IPAddr('10.0.0.1', cidr=31L)
 		self.assertEqual(res.addr, 167772160L)
 
+		self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000')
+		self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002')
+		self.assertEqual(IPAddr('xxx').hexdump, '')
+
+		self.assertEqual(IPAddr('192.0.2.0').getPTR(), '0.2.0.192.in-addr.arpa.')
+		self.assertEqual(IPAddr('192.0.2.1').getPTR(), '1.2.0.192.in-addr.arpa.')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:1946').getPTR(), 
+			'6.4.9.1.8.c.5.2.3.9.8.1.8.4.2.0.1.0.0.0.0.2.2.0.0.0.8.2.6.0.6.2.ip6.arpa.')
+
 	def testIPAddr_Equal6(self):
 		self.assertEqual(
 			IPAddr('2606:2800:220:1:248:1893::'),
 			IPAddr('2606:2800:220:1:248:1893:0:0')
 		)
+		# special case IPv6 in brackets:
+		self.assertEqual(
+			IPAddr('[2606:2800:220:1:248:1893::]'),
+			IPAddr('2606:2800:220:1:248:1893:0:0')
+		)
+
+	def testIPAddr_InInet(self):
+		ip4net = IPAddr('93.184.0.1/24')
+		ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120')
+		# ip4:
+		self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net))
+		self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net))
+		self.assertFalse(IPAddr('93.184.1.0').isInNet(ip4net))
+		self.assertFalse(IPAddr('93.184.0.1').isInNet(ip6net))
+		# ip6:
+		self.assertTrue(IPAddr('2606:2800:220:1:248:1893:25c8:1').isInNet(ip6net))
+		self.assertTrue(IPAddr('2606:2800:220:1:248:1893:25c8:ff').isInNet(ip6net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:100').isInNet(ip6net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:100').isInNet(ip4net))
+		# raw not in net:
+		self.assertFalse(IPAddr('93.184.0.1', IPAddr.CIDR_RAW).isInNet(ip4net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:1', IPAddr.CIDR_RAW).isInNet(ip6net))
+		# invalid not in net:
+		self.assertFalse(IPAddr('xxx').isInNet(ip4net))
 
 	def testIPAddr_Compare(self):
 		ip4 = [
@@ -1473,6 +1553,10 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')), 
 			'2606:28ff:220:1:248:1893:25c8:0')
 
+	def testIPAddr_CIDR_Wrong(self):
+		# too many plen representations:
+		self.assertRaises(ValueError, IPAddr, '2606:28ff:220:1:248:1893:25c8::/ffff::/::1')
+
 	def testIPAddr_CIDR_Repr(self):
 		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
 			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 9eb88c21..48b0b255 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1026,28 +1026,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				logSys.debug(l)
 		return True
 
-	def test_IPAddr(self):
-		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
-		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
-
-	def test_IPAddr_Raw(self):
-		# raw string:
-		r = IPAddr('xxx', IPAddr.CIDR_RAW)
-		self.assertFalse(r.isIPv4)
-		self.assertFalse(r.isIPv6)
-		self.assertTrue(r.isValid)
-		self.assertEqual(r, 'xxx')
-		self.assertEqual('xxx', str(r))
-		self.assertNotEqual(r, IPAddr('xxx'))
-		# raw (not IP, for example host:port as string):
-		r = IPAddr('1:2', IPAddr.CIDR_RAW)
-		self.assertFalse(r.isIPv4)
-		self.assertFalse(r.isIPv6)
-		self.assertTrue(r.isValid)
-		self.assertEqual(r, '1:2')
-		self.assertEqual('1:2', str(r))
-		self.assertNotEqual(r, IPAddr('1:2'))
-
 	def _testExecActions(self, server):
 		jails = server._Server__jails
 		for jail in jails:

From af8b650a371da68d1a35d2ad326dfddc5a6af7f6 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <info@sebres.de>
Date: Mon, 13 Jun 2016 12:56:53 +0200
Subject: [PATCH 61/61] badip timeout option introduced, set to 30 seconds in
 our test cases (#1463)

cherry-picked from 0.10 (little bit modified in test_badips.py, because no --fast option in test cases)
---
 config/action.d/badips.py              | 11 +++++++----
 fail2ban/tests/action_d/test_badips.py |  1 +
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/config/action.d/badips.py b/config/action.d/badips.py
index 025289ca..4bc879a1 100644
--- a/config/action.d/badips.py
+++ b/config/action.d/badips.py
@@ -80,14 +80,17 @@ class BadIPsAction(ActionBase):
 		If invalid `category`, `score`, `banaction` or `updateperiod`.
 	"""
 
+	TIMEOUT = 10
 	_badips = "http://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"):
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
+		self.timeout = timeout
 		self.agent = agent
 		self.category = category
 		self.score = score
@@ -119,7 +122,7 @@ class BadIPsAction(ActionBase):
 		"""
 		try:
 			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), None, 3)
+				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -173,7 +176,7 @@ class BadIPsAction(ActionBase):
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -358,7 +361,7 @@ class BadIPsAction(ActionBase):
 			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index a7f148b1..3f71b7a3 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -37,6 +37,7 @@ if sys.version_info >= (2,7):
 			self.jail.actions.add("badips", pythonModule, initOpts={
 				'category': "ssh",
 				'banaction': "test",
+				'timeout': 30,
 				})
 			self.action = self.jail.actions["badips"]