From 01e1383c9bb3b6f75f087d2d2303b37811fa5e32 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 28 Feb 2016 11:11:24 +0100
Subject: [PATCH 01/33] New class IPAddr for handling IPv4 and IPv6 addresses

---
 fail2ban/server/filter.py | 243 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 239 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index f5cb8546..da5fc1ea 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -433,7 +433,7 @@ class Filter(JailThread):
 				logSys.debug("Ignore line since time %s < %s - %s", 
 					unixTime, MyTime.time(), self.getFindTime())
 				break
-			if self.inIgnoreIPList(ip, log_ignore=True):
+			if self.inIgnoreIPList(ip.ntoa(), log_ignore=True):
 				continue
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
@@ -530,7 +530,8 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							failList.append([failRegexIndex, host, date,
+							ipaddr = IPAddr(host)
+							failList.append([failRegexIndex, ipaddr, date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
 								break
@@ -538,8 +539,9 @@ class Filter(JailThread):
 							ipMatch = DNSUtils.textToIp(host, self.__useDns)
 							if ipMatch:
 								for ip in ipMatch:
-									failList.append([failRegexIndex, ip, date,
-										 failRegex.getMatchedLines()])
+									ipaddr = IPAddr(ip)
+									failList.append([failRegexIndex, ipaddr, 
+										 date, failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
@@ -1096,3 +1098,236 @@ class DNSUtils:
 		""" Convert a binary IPv4 address into string n.n.n.n form.
 		"""
 		return socket.inet_ntoa(struct.pack("!L", ipbin))
+
+
+
+##
+# Class for IP address handling.
+#
+# This class contains methods for handling IPv4 and IPv6 addresses.
+
+class IPAddr:
+	""" provide functions to handle IPv4 and IPv6 addresses 
+	"""
+
+	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
+	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+
+	# object attributes
+	addr = 0
+	family = socket.AF_UNSPEC
+	plen = 0
+	valid = False
+	raw = ""
+
+	# object methods
+	def __init__(self, ipstring, cidr=-1):
+		""" initialize IP object by converting IP address string
+			to binary to integer
+		"""
+		for family in [socket.AF_INET, socket.AF_INET6]:
+			try:
+				binary = socket.inet_pton(family, ipstring)
+			except socket.error:
+				continue
+			else: 
+				self.valid = True
+				break
+
+		if self.valid and family == socket.AF_INET:
+			# convert host to network byte order
+			self.addr, = struct.unpack("!L", binary)
+			self.family = family
+			self.plen = 32
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+		elif self.valid and family == socket.AF_INET6:
+			# convert host to network byte order
+			hi, lo = struct.unpack("!QQ", binary)
+			self.addr = (hi << 64) | lo
+			self.family = family
+			self.plen = 128
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+			# if IPv6 address is a IPv4-compatible, make instance a IPv4
+			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
+				self.addr = lo & 0xFFFFFFFFL
+				self.family = socket.AF_INET
+				self.plen = 32
+		else:
+			# string couldn't be converted neither to a IPv4 nor
+			# to a IPv6 address - retain raw input for later use
+			# (e.g. DNS resolution)
+			self.raw = ipstring
+
+	def __repr__(self):
+		return self.ntoa()
+
+	def __str__(self):
+		return self.ntoa()
+
+	def __eq__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not self.valid and not other.valid: return self.raw == other.raw
+		if not self.valid or not other.valid: return False
+		if self.addr != other.addr: return False
+		if self.family != other.family: return False
+		if self.plen != other.plen: return False
+		return True
+
+	def __ne__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not self.valid and not other.valid: return self.raw != other.raw
+		if self.addr != other.addr: return True
+		if self.family != other.family: return True
+		if self.plen != other.plen: return True
+		return False
+
+	def __lt__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		return self.family < other.family or self.addr < other.addr
+
+	def __add__(self, other):
+		return "%s%s" % (self, other)
+
+	def __radd__(self, other):
+		return "%s%s" % (other, self)
+
+	def __hash__(self):
+		return hash(self.addr)^hash((self.plen<<16)|self.family)
+
+	def hexdump(self):
+		""" dump the ip address in as a hex sequence in
+			network byte order - for debug purpose
+		"""
+		if self.family == socket.AF_INET:
+			return "%08x" % self.addr
+		elif self.family == socket.AF_INET6:
+			return "%032x" % self.addr
+		else:
+			return ""
+	
+	def ntoa(self):
+		""" represent IP object as text like the depricated 
+			C pendant inet_ntoa() but address family independent
+		"""
+		if self.family == socket.AF_INET:
+			# convert network to host byte order
+			binary = struct.pack("!L", self.addr)
+		elif self.family == socket.AF_INET6:
+			# convert network to host byte order
+			hi = self.addr >> 64
+			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			binary = struct.pack("!QQ", hi, lo)
+		else:
+			return self.getRaw()
+
+		return socket.inet_ntop(self.family, binary)
+
+	def getPTR(self, suffix=""):
+		""" generates the DNS PTR string of the provided IP address object
+			if "suffix" is provided it will be appended as the second and top
+			level reverse domain.
+			if omitted it is implicitely set to the second and top level reverse
+			domain of the according IP address family
+		"""
+		if self.family == socket.AF_INET:
+			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+			if not suffix:
+				suffix = "in-addr.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+
+		elif self.family == socket.AF_INET6:
+			reversed_ip = ".".join(reversed(self.hexdump()))
+			if not suffix:
+				suffix =  "ip6.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+			
+		else:
+			return ""
+
+	def isIPv4(self):
+		""" return true if the IP object is of address family AF_INET
+		"""
+		return True if self.family == socket.AF_INET else False
+
+	def isIPv6(self):
+		""" return true if the IP object is of address family AF_INET6
+		"""
+		return True if self.family == socket.AF_INET6 else False
+
+	def getRaw(self):
+		""" returns the raw attribute - should only be set
+			to a non-empty string if prior address conversion
+			wasn't possible
+		"""
+		return self.raw
+
+	def isValidIP(self):
+		""" returns true if the IP object has been created
+			from a valid IP address or false if not
+		"""
+		return self.valid
+
+	
+	def isInNet(self, net):
+		""" returns true if the IP object is in the provided
+			network (object)
+		"""
+		if self.family != net.family:
+			return False
+
+		if self.family == socket.AF_INET:
+			mask = ~(0xFFFFFFFFL >> net.plen)
+
+		elif self.family == socket.AF_INET6:
+			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
+		else:
+			return False
+		
+		if self.addr & mask == net.addr:
+			return True
+
+		return False
+
+		
+	@staticmethod
+	def masktoplen(maskstr):
+		""" converts mask string to prefix length
+			only used for IPv4 masks
+		"""
+		mask = IPAddr(maskstr)
+		plen = 0
+		while mask.addr:
+			mask.addr = (mask.addr << 1) & 0xFFFFFFFFL
+			plen += 1
+		return plen
+
+
+	@staticmethod
+	def searchIP(text):
+		""" Search if an IP address if directly available and return
+			it.
+		"""
+		match = IPAddr.IP_CRE.match(text)
+		if match:
+			return match
+		else:
+			match = IPAddr.IP6_CRE.match(text)
+			if match:
+				return match
+			else:
+				return None
+

From 130874434879d2d59b4b81cd20c67be036083f77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 12:53:34 +0200
Subject: [PATCH 02/33] meantime commit: code review, simplification,
 pythonization, etc.

---
 fail2ban/server/filter.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index da5fc1ea..1b46aa2a 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -433,7 +433,7 @@ class Filter(JailThread):
 				logSys.debug("Ignore line since time %s < %s - %s", 
 					unixTime, MyTime.time(), self.getFindTime())
 				break
-			if self.inIgnoreIPList(ip.ntoa(), log_ignore=True):
+			if self.inIgnoreIPList(ip, log_ignore=True):
 				continue
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
@@ -530,8 +530,7 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							ipaddr = IPAddr(host)
-							failList.append([failRegexIndex, ipaddr, date,
+							failList.append([failRegexIndex, IPAddr(host), date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
 								break
@@ -539,9 +538,8 @@ class Filter(JailThread):
 							ipMatch = DNSUtils.textToIp(host, self.__useDns)
 							if ipMatch:
 								for ip in ipMatch:
-									ipaddr = IPAddr(ip)
-									failList.append([failRegexIndex, ipaddr, 
-										 date, failRegex.getMatchedLines()])
+									failList.append([failRegexIndex, ip, date,
+										 failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable

From a0938286023ea09f1c668ecb526f3f227e4a3d5e Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 29 Feb 2016 20:38:40 +0100
Subject: [PATCH 03/33] Make ignoreip checking address family idependent

---
 fail2ban/server/filter.py | 44 +++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 1b46aa2a..af23a1d2 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -336,7 +336,22 @@ class Filter(JailThread):
 	# when finding failures. CIDR mask and DNS are also accepted.
 	# @param ip IP address to ignore
 
-	def addIgnoreIP(self, ip):
+	def addIgnoreIP(self, ipstr):
+		# An empty string is always false
+		if ipstr == "":
+			return
+		s = ipstr.split('/', 1)
+		# IP address without CIDR mask
+		if len(s) == 1:
+			s.insert(1, -1) # <0 means no CIDR
+		elif "." in s[1]: # 255.255.255.0 style mask
+			s[1] = IPAddr.masktoplen(s[1])
+		s[1] = long(s[1])
+		
+		# Create IP address object
+		ip = IPAddr(s[0], s[1])
+
+		# log and append to ignore list
 		logSys.debug("Add " + ip + " to ignore list")
 		self.__ignoreIpList.append(ip)
 
@@ -356,34 +371,22 @@ class Filter(JailThread):
 	#
 	# Check if the given IP address matches an IP address/DNS or a CIDR
 	# mask in the ignore list.
-	# @param ip IP address
+	# @param ip IP address object
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
-		for i in self.__ignoreIpList:
-			# An empty string is always false
-			if i == "":
-				continue
-			s = i.split('/', 1)
-			# IP address without CIDR mask
-			if len(s) == 1:
-				s.insert(1, '32')
-			elif "." in s[1]: # 255.255.255.0 style mask
-				s[1] = len(re.search(
-					"(?<=b)1+", bin(DNSUtils.addr2bin(s[1]))).group())
-			s[1] = long(s[1])
-			try:
-				a = DNSUtils.addr2bin(s[0], cidr=s[1])
-				b = DNSUtils.addr2bin(ip, cidr=s[1])
-			except Exception:
+		for net in self.__ignoreIpList:
+			# if it isn't a valid IP address, try DNS resolution
+			if not net.isValidIP() and net.getRaw() != "":
 				# Check if IP in DNS
-				ips = DNSUtils.dnsToIp(i)
+				ips = DNSUtils.dnsToIp(net.getRaw())
 				if ip in ips:
 					self.logIgnoreIp(ip, log_ignore, ignore_source="dns")
 					return True
 				else:
 					continue
-			if a == b:
+			# check if the IP is covered by ignore IP
+			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
 				return True
 
@@ -391,6 +394,7 @@ class Filter(JailThread):
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
 			logSys.debug('ignore command: ' + command)
 			ret_ignore = CommandAction.executeCmd(command)
+
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
 			return ret_ignore
 

From 3893a6b780ad8271f5a10e452ccd6a6d652d51ed Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 13:50:37 +0200
Subject: [PATCH 04/33] meantime commit: code review, simplification,
 pythonization, etc.

---
 fail2ban/server/filter.py        | 3 ++-
 fail2ban/tests/servertestcase.py | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index af23a1d2..e19b322e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -375,6 +375,8 @@ class Filter(JailThread):
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
+		if isinstance(ip, basestring):
+			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
 			# if it isn't a valid IP address, try DNS resolution
 			if not net.isValidIP() and net.getRaw() != "":
@@ -394,7 +396,6 @@ class Filter(JailThread):
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
 			logSys.debug('ignore command: ' + command)
 			ret_ignore = CommandAction.executeCmd(command)
-
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
 			return ret_ignore
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 96734262..d9db6c47 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -125,14 +125,14 @@ class TransmitterBase(unittest.TestCase):
 			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdAdd, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdDel, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 
 	def jailAddDelRegexTest(self, cmd, inValues, outValues, jail):
 		cmdAdd = "add" + cmd

From a7570376717c07e783bb97ee75893f9b0713b0eb Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 29 Feb 2016 20:54:58 +0100
Subject: [PATCH 05/33] Make DNS resolution IP address family idependent

---
 fail2ban/server/filter.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e19b322e..deb47a4b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1012,18 +1012,22 @@ class DNSUtils:
 			Thanks to Kevin Drapel.
 		"""
 		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		v = DNSUtils.CACHE_nameToIp.get(dns)
-		if v is not None: 
-			return v
-		# retrieve ip (todo: use AF_INET6 for IPv6)
+		ips = DNSUtils.CACHE_nameToIp.get(dns)
+		if ips is not None: 
+			return ips
+		# retrieve ips
 		try:
-			v = set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
+			ips = list()
+			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
+				ip = IPAddr(result[4][0])
+				if ip.isValidIP():
+					ips.append(ip)
 		except socket.error, e:
 			# todo: make configurable the expired time of cache entry:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			v = list()
-		DNSUtils.CACHE_nameToIp.set(dns, v)
-		return v
+			ips = list()
+		DNSUtils.CACHE_nameToIp.set(dns, ips)
+		return ips
 
 	@staticmethod
 	def ipToName(ip):
@@ -1033,7 +1037,7 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			v = socket.gethostbyaddr(ip)[0]
+			v = socket.gethostbyaddr(ip.ntoa())[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -1068,11 +1072,11 @@ class DNSUtils:
 		"""
 		ipList = list()
 		# Search for plain IP
-		plainIP = DNSUtils.searchIP(text)
+		plainIP = IPAddr.searchIP(text)
 		if not plainIP is None:
-			plainIPStr = plainIP.group(0)
-			if DNSUtils.isValidIP(plainIPStr):
-				ipList.append(plainIPStr)
+			ip = IPAddr(plainIP.group(0))
+			if ip.isValidIP():
+				ipList.append(ip)
 
 		# If we are allowed to resolve -- give it a try if nothing was found
 		if useDns in ("yes", "warn") and not ipList:

From 07c9f38e4529cc45e3336d30e66d3c4c195fefff Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 14:02:03 +0200
Subject: [PATCH 06/33] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/server/filter.py        | 7 +++++--
 fail2ban/tests/filtertestcase.py | 6 +++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index deb47a4b..3682125c 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -375,7 +375,7 @@ class Filter(JailThread):
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
-		if isinstance(ip, basestring):
+		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
 			# if it isn't a valid IP address, try DNS resolution
@@ -1037,7 +1037,10 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			v = socket.gethostbyaddr(ip.ntoa())[0]
+			if not isinstance(ip, IPAddr):
+				v = socket.gethostbyaddr(ip)[0]
+			else:
+				v = socket.gethostbyaddr(ip.ntoa())[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ae22b88f..b5a772d9 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1314,9 +1314,9 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1328,7 +1328,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		for s in hostnames:
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
-				self.assertEqual(res, ['93.184.216.34'])
+				self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 

From 85b895178b8a6dd9ce7c9086c8c451b72efdb4c1 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Wed, 2 Mar 2016 06:56:57 +0100
Subject: [PATCH 07/33] change IP address string to object handling part 1

# Conflicts:
#	fail2ban/server/filter.py
---
 fail2ban/server/actions.py    |  9 ++++++---
 fail2ban/server/banmanager.py |  6 +++---
 fail2ban/server/database.py   | 16 ++++++++++------
 fail2ban/server/filter.py     | 18 ++++++++++++------
 fail2ban/server/ticket.py     |  3 ---
 5 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 5469722f..428d1ccf 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -42,6 +42,7 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
+from .filter import IPAddr
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -180,7 +181,7 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def removeBannedIP(self, ip):
+	def removeBannedIP(self, ipstr):
 		"""Removes banned IP calling actions' unban method
 
 		Remove a banned IP now, rather than waiting for it to expire,
@@ -188,14 +189,16 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ip : str
-			The IP address to unban
+		ipstr : str
+			The IP address string to unban
 
 		Raises
 		------
 		ValueError
 			If `ip` is not banned
 		"""
+		# Create new IPAddr object from IP string
+		ip = IPAddr(ipstr)
 		# Always delete ip from database (also if currently not banned)
 		if self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 0ee028ef..67ae1b71 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -152,9 +152,9 @@ class BanManager:
 			for banData in self.__banList:
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
-				# TODO: IPv6 compatibility
-				reversed_ip = ".".join(reversed(ip.split(".")))
-				question = "%s.origin.asn.cymru.com" % reversed_ip
+				question = ip.getPTR("origin.asn.cymru.com" if ip.isIPv4()
+						else "origin6.asn.cymru.com"
+				)
 				try:
 					answers = dns.resolver.query(question, "TXT")
 					for rdata in answers:
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index b7fd4d47..e5e51992 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -32,6 +32,7 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
+from .filter import IPAddr
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -422,7 +423,7 @@ class Fail2BanDb(object):
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ticket.getIP(), int(round(ticket.getTime())),
+			(jail.name, ticket.getIP().ntoa(), int(round(ticket.getTime())),
 				ticket.getData()))
 
 	@commitandrollback
@@ -436,7 +437,7 @@ class Fail2BanDb(object):
 		ip : str
 			IP to be removed.
 		"""
-		queryArgs = (jail.name, ip);
+		queryArgs = (jail.name, ip.ntoa());
 		cur.execute(
 			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
 			queryArgs);
@@ -454,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip)
+			queryArgs.append(ip.ntoa())
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)
@@ -470,7 +471,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now.  Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : str
+		ip : IPAddr object
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -479,7 +480,8 @@ class Fail2BanDb(object):
 			List of `Ticket`s for bans stored in database.
 		"""
 		tickets = []
-		for ip, timeofban, data in self._getBans(**kwargs):
+		for ipstr, timeofban, data in self._getBans(**kwargs):
+			ip = IPAddr(ipstr)
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban))
 			tickets[-1].setData(data)
@@ -499,7 +501,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now. Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : str
+		ip : IPAddr object
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -520,6 +522,8 @@ class Fail2BanDb(object):
 			ticket = None
 
 			results = list(self._getBans(ip=ip, jail=jail, bantime=bantime))
+			# Convert IP strings to IPAddr objects
+			results = map(lambda i:(IPAddr(i[0]),)+i[1:], results)
 			if results:
 				prev_banip = results[0][0]
 				matches = []
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 3682125c..371a13d7 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -306,13 +306,19 @@ class Filter(JailThread):
 	def getIgnoreCommand(self):
 		return self.__ignoreCommand
 
+	##
+	# create new IPAddr object from IP address string
+	def newIP(self, ipstr):
+		return IPAddr(ipstr)
+
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
 	#
 	# to enable banip fail2ban-client BAN command
 
-	def addBannedIP(self, ip):
+	def addBannedIP(self, ipstr):
+		ip = IPAddr(ipstr)
 		if self.inIgnoreIPList(ip):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
@@ -540,11 +546,11 @@ class Filter(JailThread):
 							if not checkAllRegex:
 								break
 						else:
-							ipMatch = DNSUtils.textToIp(host, self.__useDns)
-							if ipMatch:
-								for ip in ipMatch:
-									failList.append([failRegexIndex, ip, date,
-										 failRegex.getMatchedLines()])
+							ips = DNSUtils.textToIp(host, self.__useDns)
+							if ips:
+								for ip in ips:
+									failList.append([failRegexIndex, ip, 
+										 date, failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index b3eea052..7b7eb908 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -72,9 +72,6 @@ class Ticket:
 			return False
 
 	def setIP(self, value):
-		if isinstance(value, basestring):
-			# guarantee using regular str instead of unicode for the IP
-			value = str(value)
 		self.__ip = value
 	
 	def getIP(self):

From afe1f73af27503972a4843a770adf3e46f207cde Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 19:44:52 +0200
Subject: [PATCH 08/33] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed) unnecessarily code aggravation with
 explicit converting reverted - implicit converting inside internal functions
 if not IPAddr object;

---
 fail2ban/server/actions.py            |   8 +-
 fail2ban/server/database.py           |  21 ++--
 fail2ban/server/failmanager.py        |   9 ++
 fail2ban/server/filter.py             | 100 +++++++++--------
 fail2ban/server/ticket.py             |   4 +
 fail2ban/tests/failmanagertestcase.py |   3 +-
 fail2ban/tests/filtertestcase.py      | 153 ++++++++++++++++++++++----
 7 files changed, 212 insertions(+), 86 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 428d1ccf..1b72af91 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -181,7 +181,7 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def removeBannedIP(self, ipstr):
+	def removeBannedIP(self, ip):
 		"""Removes banned IP calling actions' unban method
 
 		Remove a banned IP now, rather than waiting for it to expire,
@@ -189,16 +189,14 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ipstr : str
-			The IP address string to unban
+		ip : str or IPAddr
+			The IP address  to unban
 
 		Raises
 		------
 		ValueError
 			If `ip` is not banned
 		"""
-		# Create new IPAddr object from IP string
-		ip = IPAddr(ipstr)
 		# Always delete ip from database (also if currently not banned)
 		if self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index e5e51992..6a3d87c3 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -32,7 +32,6 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
-from .filter import IPAddr
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -412,18 +411,19 @@ class Fail2BanDb(object):
 		ticket : BanTicket
 			Ticket of the ban to be added.
 		"""
+		ip = str(ticket.getIP())
 		try:
-			del self._bansMergedCache[(ticket.getIP(), jail)]
+			del self._bansMergedCache[(ip, jail)]
 		except KeyError:
 			pass
 		try:
-			del self._bansMergedCache[(ticket.getIP(), None)]
+			del self._bansMergedCache[(ip, None)]
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ticket.getIP().ntoa(), int(round(ticket.getTime())),
+			(jail.name, ip, int(round(ticket.getTime())),
 				ticket.getData()))
 
 	@commitandrollback
@@ -437,7 +437,7 @@ class Fail2BanDb(object):
 		ip : str
 			IP to be removed.
 		"""
-		queryArgs = (jail.name, ip.ntoa());
+		queryArgs = (jail.name, str(ip));
 		cur.execute(
 			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
 			queryArgs);
@@ -455,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip.ntoa())
+			queryArgs.append(ip)
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)
@@ -471,7 +471,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now.  Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : IPAddr object
+		ip : str
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -480,8 +480,7 @@ class Fail2BanDb(object):
 			List of `Ticket`s for bans stored in database.
 		"""
 		tickets = []
-		for ipstr, timeofban, data in self._getBans(**kwargs):
-			ip = IPAddr(ipstr)
+		for ip, timeofban, data in self._getBans(**kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban))
 			tickets[-1].setData(data)
@@ -501,7 +500,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now. Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : IPAddr object
+		ip : str
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -522,8 +521,6 @@ class Fail2BanDb(object):
 			ticket = None
 
 			results = list(self._getBans(ip=ip, jail=jail, bantime=bantime))
-			# Convert IP strings to IPAddr objects
-			results = map(lambda i:(IPAddr(i[0]),)+i[1:], results)
 			if results:
 				prev_banip = results[0][0]
 				matches = []
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index ae97b36a..b342b280 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -54,6 +54,15 @@ class FailManager:
 		with self.__lock:
 			return self.__failTotal
 	
+	def getFailCount(self):
+		# may be slow on large list of failures, should be used for test purposes only...
+		with self.__lock:
+			return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()])
+
+	def getFailTotal(self):
+		with self.__lock:
+			return self.__failTotal
+
 	def setMaxRetry(self, value):
 		self.__maxRetry = value
 	
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 371a13d7..04f34821 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -306,19 +306,15 @@ class Filter(JailThread):
 	def getIgnoreCommand(self):
 		return self.__ignoreCommand
 
-	##
-	# create new IPAddr object from IP address string
-	def newIP(self, ipstr):
-		return IPAddr(ipstr)
-
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
 	#
 	# to enable banip fail2ban-client BAN command
 
-	def addBannedIP(self, ipstr):
-		ip = IPAddr(ipstr)
+	def addBannedIP(self, ip):
+		if not isinstance(ip, IPAddr):
+			ip = IPAddr(ip)
 		if self.inIgnoreIPList(ip):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
@@ -358,11 +354,11 @@ class Filter(JailThread):
 		ip = IPAddr(s[0], s[1])
 
 		# log and append to ignore list
-		logSys.debug("Add " + ip + " to ignore list")
+		logSys.debug("Add %r to ignore list (%r, %r)", ip, s[0], s[1])
 		self.__ignoreIpList.append(ip)
 
 	def delIgnoreIP(self, ip):
-		logSys.debug("Remove " + ip + " from ignore list")
+		logSys.debug("Remove %r from ignore list", ip)
 		self.__ignoreIpList.remove(ip)
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
@@ -384,18 +380,9 @@ class Filter(JailThread):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
-			# if it isn't a valid IP address, try DNS resolution
-			if not net.isValidIP() and net.getRaw() != "":
-				# Check if IP in DNS
-				ips = DNSUtils.dnsToIp(net.getRaw())
-				if ip in ips:
-					self.logIgnoreIp(ip, log_ignore, ignore_source="dns")
-					return True
-				else:
-					continue
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
-				self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
+				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValidIP() else "dns"))
 				return True
 
 		if self.__ignoreCommand:
@@ -1006,8 +993,6 @@ from .utils import Utils
 
 class DNSUtils:
 
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
 	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
@@ -1053,17 +1038,6 @@ class DNSUtils:
 		DNSUtils.CACHE_ipToName.set(ip, v)
 		return v
 
-	@staticmethod
-	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
-		"""
-		match = DNSUtils.IP_CRE.match(text)
-		if match:
-			return match
-		else:
-			return None
-
 	@staticmethod
 	def isValidIP(string):
 		""" Return true if str is a valid IP
@@ -1082,7 +1056,7 @@ class DNSUtils:
 		ipList = list()
 		# Search for plain IP
 		plainIP = IPAddr.searchIP(text)
-		if not plainIP is None:
+		if plainIP is not None:
 			ip = IPAddr(plainIP.group(0))
 			if ip.isValidIP():
 				ipList.append(ip)
@@ -1122,7 +1096,7 @@ class DNSUtils:
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
 
-class IPAddr:
+class IPAddr(object):
 	""" provide functions to handle IPv4 and IPv6 addresses 
 	"""
 
@@ -1136,8 +1110,22 @@ class IPAddr:
 	valid = False
 	raw = ""
 
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
+
 	# object methods
-	def __init__(self, ipstring, cidr=-1):
+	def __init(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -1193,7 +1181,9 @@ class IPAddr:
 		return self.ntoa()
 
 	def __eq__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
 		if not self.valid and not other.valid: return self.raw == other.raw
 		if not self.valid or not other.valid: return False
 		if self.addr != other.addr: return False
@@ -1202,7 +1192,9 @@ class IPAddr:
 		return True
 
 	def __ne__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return True
+			other = IPAddr(other)
 		if not self.valid and not other.valid: return self.raw != other.raw
 		if self.addr != other.addr: return True
 		if self.family != other.family: return True
@@ -1210,7 +1202,9 @@ class IPAddr:
 		return False
 
 	def __lt__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
@@ -1220,7 +1214,9 @@ class IPAddr:
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		return hash(self.addr)^hash((self.plen<<16)|self.family)
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa())
+		#return hash(self.addr)^hash((self.plen<<16)|self.family)
 
 	def hexdump(self):
 		""" dump the ip address in as a hex sequence in
@@ -1302,6 +1298,11 @@ class IPAddr:
 		""" returns true if the IP object is in the provided
 			network (object)
 		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValidIP() and net.getRaw() != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.getRaw())
+
 		if self.family != net.family:
 			return False
 
@@ -1318,18 +1319,27 @@ class IPAddr:
 
 		return False
 
+	@property
+	def maskplen(self):
+		plen = 0
+		if (hasattr(self, '_maskplen')):
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+
 		
 	@staticmethod
 	def masktoplen(maskstr):
 		""" converts mask string to prefix length
 			only used for IPv4 masks
 		"""
-		mask = IPAddr(maskstr)
-		plen = 0
-		while mask.addr:
-			mask.addr = (mask.addr << 1) & 0xFFFFFFFFL
-			plen += 1
-		return plen
+		return IPAddr(maskstr).maskplen
 
 
 	@staticmethod
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 7b7eb908..3307a6c9 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -72,6 +72,10 @@ class Ticket:
 			return False
 
 	def setIP(self, value):
+		# guarantee using IPAddr instead of unicode, str for the IP
+		if isinstance(value, basestring):
+			from .filter import IPAddr
+			value = IPAddr(value)
 		self.__ip = value
 	
 	def getIP(self):
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index a8a71723..36bc87a3 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -28,6 +28,7 @@ import unittest
 
 from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
+from ..server.filter import IPAddr
 from ..server.ticket import FailTicket
 
 
@@ -140,7 +141,7 @@ class AddFailure(unittest.TestCase):
 		#ticket = FailTicket('193.168.0.128', None)
 		ticket = self.__failManager.toBan()
 		self.assertEqual(ticket.getIP(), "193.168.0.128")
-		self.assertTrue(isinstance(ticket.getIP(), str))
+		self.assertTrue(isinstance(ticket.getIP(), (str, IPAddr)))
 
 		# finish with rudimentary tests of the ticket
 		# verify consistent str
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index b5a772d9..7198185a 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -38,7 +38,7 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils
+from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils, IPAddr
 from ..server.failmanager import FailManagerEmpty
 from ..server.mytime import MyTime
 from ..server.utils import Utils
@@ -154,19 +154,41 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None):
 
 	Test filter to contain target ticket
 	"""
+	# one or multiple tickets:
+	if not isinstance(output[0], (tuple,list)):
+		tickcount = 1
+		failcount = (count if count else output[1])
+	else:
+		tickcount = len(output)
+		failcount = (count if count else sum((o[1] for o in output)))
+
+	found = []
 	if isinstance(filter_, DummyJail):
 		# get fail ticket from jail
-		found = _ticket_tuple(filter_.getFailTicket())
+		found.append(_ticket_tuple(filter_.getFailTicket()))
 	else:
 		# when we are testing without jails
 		# wait for failures (up to max time)
 		Utils.wait_for(
-			lambda: filter_.failManager.getFailTotal() >= (count if count else output[1]),
+			lambda: filter_.failManager.getFailCount() >= (tickcount, failcount),
 			_maxWaitTime(10))
-		# get fail ticket from filter
-		found = _ticket_tuple(filter_.failManager.toBan())
+		# get fail ticket(s) from filter
+		while tickcount:
+			try:
+				found.append(_ticket_tuple(filter_.failManager.toBan()))
+			except FailManagerEmpty:
+				break
+			tickcount -= 1
 
-	_assert_equal_entries(utest, found, output, count)
+	if not isinstance(output[0], (tuple,list)):
+		utest.assertEqual(len(found), 1)
+		_assert_equal_entries(utest, found[0], output, count)
+	else:
+		# sort by string representation of ip (multiple failures with different ips):
+		found = sorted(found, key=lambda x: str(x))
+		output = sorted(output, key=lambda x: str(x))
+		for f, o in zip(found, output):
+			_assert_equal_entries(utest, f, o)
 
 
 def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""):
@@ -315,6 +337,10 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.1.255'))
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.0.255'))
 
+	def testWrongIPMask(self):
+		self.filter.addIgnoreIP('192.168.1.0/255.255.0.0')
+		self.assertRaises(ValueError, self.filter.addIgnoreIP, '192.168.1.0/255.255.0.128')
+
 	def testIgnoreInProcessLine(self):
 		setUpMyTime()
 		self.filter.addIgnoreIP('192.168.1.0/25')
@@ -345,16 +371,21 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertNotLogged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED"))
 
 
-class IgnoreIPDNS(IgnoreIP):
+class IgnoreIPDNS(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
-		IgnoreIP.setUp(self)
+		LogCaptureTestCase.setUp(self)
+		self.jail = DummyJail()
+		self.filter = FileFilter(self.jail)
 
 	def testIgnoreIPDNSOK(self):
 		self.filter.addIgnoreIP("www.epfl.ch")
 		self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12"))
+		self.filter.addIgnoreIP("example.com")
+		self.assertTrue(self.filter.inIgnoreIPList("93.184.216.34"))
+		self.assertTrue(self.filter.inIgnoreIPList("2606:2800:220:1:248:1893:25c8:1946"))
 
 	def testIgnoreIPDNSNOK(self):
 		# Test DNS
@@ -1109,18 +1140,18 @@ class GetFailures(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures04(self):
-		output = [('212.41.96.186', 4, 1124013600.0),
-				  ('212.41.96.185', 4, 1124017198.0)]
+		# because of not exact time in testcase04.log (no year), we should always use our test time:
+		self.assertEqual(MyTime.time(), 1124013600)
+		# should find exact 4 failures for *.186 and 2 failures for *.185
+		output = (('212.41.96.186', 4, 1124013600.0),
+				  ('212.41.96.185', 2, 1124013598.0))
 
+		self.filter.setMaxRetry(2)
 		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_04)
 
-		try:
-			for i, out in enumerate(output):
-				_assert_correct_last_attempt(self, self.filter, out)
-		except FailManagerEmpty:
-			pass
+		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailuresWrongChar(self):
 		# write wrong utf-8 char:
@@ -1160,20 +1191,31 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresUseDNS(self):
 		unittest.F2B.SkipIfNoNetwork()
 		# We should still catch failures with usedns = no ;-)
-		output_yes = ('93.184.216.34', 2, 1124013539.0,
-					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
-					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_yes = (
+			('93.184.216.34', 2, 1124013539.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
+			   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			),
+			('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2']
+			),
+		)
 
-		output_no = ('93.184.216.34', 1, 1124013539.0,
-					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_no = (
+			('93.184.216.34', 1, 1124013539.0,
+			  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			)
+		)
 
 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,
 		#				  FileFilter, None, useDns='wrong_value_for_useDns')
 
-		for useDns, output in (('yes',  output_yes),
-							   ('no',   output_no),
-							   ('warn', output_yes)):
+		for useDns, output in (
+			('yes',  output_yes),
+			('no',   output_no),
+			('warn', output_yes)
+		):
 			jail = DummyJail()
 			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.active = True
@@ -1356,6 +1398,71 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.bin2addr(167772160L)
 		self.assertEqual(res, '10.0.0.0')
 
+	def testIPAddr_Equal6(self):
+		self.assertEqual(
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:0:0')
+		)
+
+	def testIPAddr_Compare(self):
+		ip4 = [
+			IPAddr('93.184.0.1'),
+			IPAddr('93.184.216.1'),
+			IPAddr('93.184.216.34')
+		]
+		ip6 = [
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:0'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:1946')
+		]
+		# ip4
+		self.assertNotEqual(ip4[0], None)
+		self.assertTrue(ip4[0] is not None)
+		self.assertFalse(ip4[0] is None)
+		self.assertLess(None, ip4[0])
+		self.assertLess(ip4[0], ip4[1])
+		self.assertLess(ip4[1], ip4[2])
+		self.assertEqual(sorted(reversed(ip4)), ip4)
+		# ip6
+		self.assertNotEqual(ip6[0], None)
+		self.assertTrue(ip6[0] is not None)
+		self.assertFalse(ip6[0] is None)
+		self.assertLess(None, ip6[0])
+		self.assertLess(ip6[0], ip6[1])
+		self.assertLess(ip6[1], ip6[2])
+		self.assertEqual(sorted(reversed(ip6)), ip6)
+		# ip4 vs ip6
+		self.assertNotEqual(ip4[0], ip6[0])
+		self.assertLess(ip4[0], ip6[0])
+		self.assertLess(ip4[2], ip6[2])
+		self.assertEqual(sorted(reversed(ip4+ip6)), ip4+ip6)
+		# hashing (with string as key):
+		d={
+			'93.184.216.34': 'ip4-test', 
+			'2606:2800:220:1:248:1893:25c8:1946': 'ip6-test'
+		}
+		d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()])
+		self.assertTrue(isinstance(d.keys()[0], basestring))
+		self.assertTrue(isinstance(d2.keys()[0], IPAddr))
+		self.assertEqual(d.get(ip4[2], ''), 'ip4-test')
+		self.assertEqual(d.get(ip6[2], ''), 'ip6-test')
+		self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test')
+		self.assertEqual(d2.get(str(ip6[2]), ''), 'ip6-test')
+		# compare with string direct:
+		self.assertEqual(d, d2)
+
+	def testIPAddr_CompareDNS(self):
+		ips = IPAddr('example.com')
+		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))
+		self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips))
+
+	def testIPAddr_Cached(self):
+		ips = [DNSUtils.dnsToIp('example.com'), DNSUtils.dnsToIp('example.com')]
+		for ip1, ip2 in zip(ips, ips):
+			self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('93.184.216.34'); ip2 = IPAddr('93.184.216.34'); self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2))
+
 
 class JailTests(unittest.TestCase):
 

From d125f882d4fd1d1d9e096aa328cdb3eec6abac5e Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 5 Mar 2016 12:02:36 +0100
Subject: [PATCH 09/33] explicitely treat join arguments as string in
 beautifier

---
 fail2ban/client/beautifier.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 812fbe65..308cb035 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -78,13 +78,13 @@ class Beautifier:
 						prefix1 = "   " if n == len(response) - 1 else "|  "
 						for m, res2 in enumerate(res1[1]):
 							prefix2 = prefix1 + ("`-" if m == len(res1[1]) - 1 else "|-")
-							val = " ".join(res2[1]) if isinstance(res2[1], list) else res2[1]
+							val = " ".join(map(str, res2[1])) if isinstance(res2[1], list) else res2[1]
 							msg.append("%s %s:\t%s" % (prefix2, res2[0], val))
 				else:
 					msg = ["Status"]
 					for n, res1 in enumerate(response):
 						prefix1 = "`-" if n == len(response) - 1 else "|-"
-						val = " ".join(res1[1]) if isinstance(res1[1], list) else res1[1]
+						val = " ".join(map(str, res1[1])) if isinstance(res1[1], list) else res1[1]
 						msg.append("%s %s:\t%s" % (prefix1, res1[0], val))
 				msg = "\n".join(msg)
 			elif inC[1] == "syslogsocket":

From db9f3f738f88ef0f7ae535a578055666f1843292 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 23:42:06 +0100
Subject: [PATCH 10/33] add ip6-loopback to default ignoreip statement

---
 config/jail.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index b41ce24c..36c4eecb 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -47,7 +47,7 @@ before = paths-debian.conf
 # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
 # ban a host which matches an address in this list. Several addresses can be
 # defined using space (and/or comma) separator.
-ignoreip = 127.0.0.1/8
+ignoreip = 127.0.0.1/8 ::1
 
 # External command that will take an tagged arguments to ignore, e.g. <ip>,
 # and return true if the IP is to be ignored. False otherwise.

From ce196744d1d509c895995c759ee2f4d014195ac1 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 23:29:12 +0100
Subject: [PATCH 11/33] Update ChangeLog / THANKS entries

---
 ChangeLog | 8 ++++++++
 THANKS    | 1 +
 2 files changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 033cd9ec..0bebb471 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -78,6 +78,14 @@ ver. 0.9.4 (2016/03/08) - for-you-ladies
    * sshd filter got new failregex to match "maximum authentication
      attempts exceeded" (introduced in openssh 6.8)
    * Added filter for Mac OS screen sharing (VNC) daemon
+   * IPv6 support:
+     - IP addresses are now handled as objects rather than strings capable for 
+       handling both address types IPv4 and IPv6
+     - iptables related actions have been amended to support IPv6 specific actions
+       additionally
+     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
+     - pf action for *BSD systems has been improved and supports now also v4 and v6
+     - Name resolution is now working for either address type
 
 - Enhancements:
    * Do not rotate empty log files
diff --git a/THANKS b/THANKS
index 64de43a4..c8a019a6 100644
--- a/THANKS
+++ b/THANKS
@@ -12,6 +12,7 @@ Adrien Clerc
 ache
 ag4ve (Shawn)
 Alasdair D. Campbell
+Alexander Koeppe (IPv6 support)
 Alexandre Perrin (kAworu)
 Amir Caspi
 Amy

From 9ede535a616802434ecb9826ed5c40a6ac81317f Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 15 Mar 2016 00:36:08 +0100
Subject: [PATCH 12/33] remove obsolete IP related code from DNSUtils class

# Conflicts:
#	fail2ban/server/filter.py
---
 fail2ban/server/filter.py | 36 +++---------------------------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 04f34821..baf4bc9b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -981,10 +981,10 @@ class JournalFilter(Filter): # pragma: systemd no cover
 		return []
 
 ##
-# Utils class for DNS and IP handling.
+# Utils class for DNS handling.
+#
+# This class contains only static methods used to handle DNS 
 #
-# This class contains only static methods used to handle DNS and IP
-# addresses.
 
 import socket
 import struct
@@ -1038,17 +1038,6 @@ class DNSUtils:
 		DNSUtils.CACHE_ipToName.set(ip, v)
 		return v
 
-	@staticmethod
-	def isValidIP(string):
-		""" Return true if str is a valid IP
-		"""
-		s = string.split('/', 1)
-		try:
-			socket.inet_aton(s[0])
-			return True
-		except socket.error: # pragma: no cover
-			return False
-
 	@staticmethod
 	def textToIp(text, useDns):
 		""" Return the IP of DNS found in a given text.
@@ -1072,25 +1061,6 @@ class DNSUtils:
 
 		return ipList
 
-	@staticmethod
-	def addr2bin(ipstring, cidr=None):
-		""" Convert a string IPv4 address into binary form.
-		If cidr is supplied, return the network address for the given block
-		"""
-		if cidr is None:
-			return struct.unpack("!L", socket.inet_aton(ipstring))[0]
-		else:
-			MASK = 0xFFFFFFFFL
-			return ~(MASK >> cidr) & MASK & DNSUtils.addr2bin(ipstring)
-
-	@staticmethod
-	def bin2addr(ipbin):
-		""" Convert a binary IPv4 address into string n.n.n.n form.
-		"""
-		return socket.inet_ntoa(struct.pack("!L", ipbin))
-
-
-
 ##
 # Class for IP address handling.
 #

From 6985531e91085c0daa7aa746c909a1293bc91bc2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 9 May 2016 16:23:59 +0200
Subject: [PATCH 13/33] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/server/filter.py        |  9 ++++++---
 fail2ban/tests/filtertestcase.py | 24 ++++++++++--------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index baf4bc9b..b808a6e3 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1102,11 +1102,10 @@ class IPAddr(object):
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
-			except socket.error:
-				continue
-			else: 
 				self.valid = True
 				break
+			except socket.error:
+				continue
 
 		if self.valid and family == socket.AF_INET:
 			# convert host to network byte order
@@ -1178,9 +1177,13 @@ class IPAddr(object):
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 7198185a..371db3bc 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1383,20 +1383,16 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(res, None)
 
 	def testAddr2bin(self):
-		res = DNSUtils.addr2bin('10.0.0.0')
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=None)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=32L)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=32L)
-		self.assertEqual(res, 167772161L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=31L)
-		self.assertEqual(res, 167772160L)
-
-	def testBin2addr(self):
-		res = DNSUtils.bin2addr(167772160L)
-		self.assertEqual(res, '10.0.0.0')
+		res = IPAddr('10.0.0.0')
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=None)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=32L)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.1', cidr=32L)
+		self.assertEqual(res.addr, 167772161L)
+		res = IPAddr('10.0.0.1', cidr=31L)
+		self.assertEqual(res.addr, 167772160L)
 
 	def testIPAddr_Equal6(self):
 		self.assertEqual(

From 8cb4a3f59ef1a8078ef6919ce587bb6ebed66711 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 9 May 2016 17:00:35 +0200
Subject: [PATCH 14/33] move DNTUtils, IPAddr related code to dedicated source
 file ipdns.py (also resolves some cyclic import references)

---
 MANIFEST                                      |   1 +
 .../ignorecommands/apache-fakegooglebot       |   4 +-
 fail2ban/server/filter.py                     | 351 +---------------
 fail2ban/server/ipdns.py                      | 389 ++++++++++++++++++
 fail2ban/server/ticket.py                     |   2 +-
 fail2ban/tests/failmanagertestcase.py         |   2 +-
 fail2ban/tests/filtertestcase.py              |   3 +-
 fail2ban/tests/utils.py                       |   2 +-
 8 files changed, 398 insertions(+), 356 deletions(-)
 create mode 100644 fail2ban/server/ipdns.py

diff --git a/MANIFEST b/MANIFEST
index 4ebf3fad..3a5b4477 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -164,6 +164,7 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
+fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index 19fb5107..86a28eaa 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -14,7 +14,7 @@ def process_args(argv):
 
     ip = argv[1]
 
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
     if not DNSUtils.isValidIP(ip):
        sys.stderr.write("Argument must be a single valid IP. Got: %s\n"
                         % ip)
@@ -23,7 +23,7 @@ def process_args(argv):
 
 def is_googlebot(ip):
     import re
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
 
     host = DNSUtils.ipToName(ip)
     if not host or not re.match('.*\.google(bot)?\.com$', host):
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b808a6e3..49aba907 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -31,6 +31,7 @@ import re
 import sys
 
 from .failmanager import FailManagerEmpty, FailManager
+from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
@@ -980,353 +981,3 @@ class JournalFilter(Filter): # pragma: systemd no cover
 	def getJournalMatch(self, match): # pragma: no cover - Base class, not used
 		return []
 
-##
-# Utils class for DNS handling.
-#
-# This class contains only static methods used to handle DNS 
-#
-
-import socket
-import struct
-from .utils import Utils
-
-
-class DNSUtils:
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
-	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	@staticmethod
-	def dnsToIp(dns):
-		""" Convert a DNS into an IP address using the Python socket module.
-			Thanks to Kevin Drapel.
-		"""
-		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		ips = DNSUtils.CACHE_nameToIp.get(dns)
-		if ips is not None: 
-			return ips
-		# retrieve ips
-		try:
-			ips = list()
-			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
-				ip = IPAddr(result[4][0])
-				if ip.isValidIP():
-					ips.append(ip)
-		except socket.error, e:
-			# todo: make configurable the expired time of cache entry:
-			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			ips = list()
-		DNSUtils.CACHE_nameToIp.set(dns, ips)
-		return ips
-
-	@staticmethod
-	def ipToName(ip):
-		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
-		v = DNSUtils.CACHE_ipToName.get(ip, ())
-		if v != ():
-			return v
-		# retrieve name
-		try:
-			if not isinstance(ip, IPAddr):
-				v = socket.gethostbyaddr(ip)[0]
-			else:
-				v = socket.gethostbyaddr(ip.ntoa())[0]
-		except socket.error, e:
-			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
-			v = None
-		DNSUtils.CACHE_ipToName.set(ip, v)
-		return v
-
-	@staticmethod
-	def textToIp(text, useDns):
-		""" Return the IP of DNS found in a given text.
-		"""
-		ipList = list()
-		# Search for plain IP
-		plainIP = IPAddr.searchIP(text)
-		if plainIP is not None:
-			ip = IPAddr(plainIP.group(0))
-			if ip.isValidIP():
-				ipList.append(ip)
-
-		# If we are allowed to resolve -- give it a try if nothing was found
-		if useDns in ("yes", "warn") and not ipList:
-			# Try to get IP from possible DNS
-			ip = DNSUtils.dnsToIp(text)
-			ipList.extend(ip)
-			if ip and useDns == "warn":
-				logSys.warning("Determined IP using DNS Lookup: %s = %s",
-					text, ipList)
-
-		return ipList
-
-##
-# Class for IP address handling.
-#
-# This class contains methods for handling IPv4 and IPv6 addresses.
-
-class IPAddr(object):
-	""" provide functions to handle IPv4 and IPv6 addresses 
-	"""
-
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
-
-	# object attributes
-	addr = 0
-	family = socket.AF_UNSPEC
-	plen = 0
-	valid = False
-	raw = ""
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
-		ip = IPAddr.CACHE_OBJ.get(args)
-		if ip is not None:
-			return ip
-		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
-		return ip
-
-	# object methods
-	def __init(self, ipstring, cidr=-1):
-		""" initialize IP object by converting IP address string
-			to binary to integer
-		"""
-		for family in [socket.AF_INET, socket.AF_INET6]:
-			try:
-				binary = socket.inet_pton(family, ipstring)
-				self.valid = True
-				break
-			except socket.error:
-				continue
-
-		if self.valid and family == socket.AF_INET:
-			# convert host to network byte order
-			self.addr, = struct.unpack("!L", binary)
-			self.family = family
-			self.plen = 32
-
-			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
-				mask = ~(0xFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
-
-		elif self.valid and family == socket.AF_INET6:
-			# convert host to network byte order
-			hi, lo = struct.unpack("!QQ", binary)
-			self.addr = (hi << 64) | lo
-			self.family = family
-			self.plen = 128
-
-			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
-				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
-
-			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
-				self.addr = lo & 0xFFFFFFFFL
-				self.family = socket.AF_INET
-				self.plen = 32
-		else:
-			# string couldn't be converted neither to a IPv4 nor
-			# to a IPv6 address - retain raw input for later use
-			# (e.g. DNS resolution)
-			self.raw = ipstring
-
-	def __repr__(self):
-		return self.ntoa()
-
-	def __str__(self):
-		return self.ntoa()
-
-	def __eq__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw == other.raw
-		if not self.valid or not other.valid: return False
-		if self.addr != other.addr: return False
-		if self.family != other.family: return False
-		if self.plen != other.plen: return False
-		return True
-
-	def __ne__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return True
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw != other.raw
-		if self.addr != other.addr: return True
-		if self.family != other.family: return True
-		if self.plen != other.plen: return True
-		return False
-
-	def __lt__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		return self.family < other.family or self.addr < other.addr
-
-	def __add__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
-		return "%s%s" % (self, other)
-
-	def __radd__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
-		return "%s%s" % (other, self)
-
-	def __hash__(self):
-		# should be the same as by string (because of possible compare with string):
-		return hash(self.ntoa())
-		#return hash(self.addr)^hash((self.plen<<16)|self.family)
-
-	def hexdump(self):
-		""" dump the ip address in as a hex sequence in
-			network byte order - for debug purpose
-		"""
-		if self.family == socket.AF_INET:
-			return "%08x" % self.addr
-		elif self.family == socket.AF_INET6:
-			return "%032x" % self.addr
-		else:
-			return ""
-	
-	def ntoa(self):
-		""" represent IP object as text like the depricated 
-			C pendant inet_ntoa() but address family independent
-		"""
-		if self.family == socket.AF_INET:
-			# convert network to host byte order
-			binary = struct.pack("!L", self.addr)
-		elif self.family == socket.AF_INET6:
-			# convert network to host byte order
-			hi = self.addr >> 64
-			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
-			binary = struct.pack("!QQ", hi, lo)
-		else:
-			return self.getRaw()
-
-		return socket.inet_ntop(self.family, binary)
-
-	def getPTR(self, suffix=""):
-		""" generates the DNS PTR string of the provided IP address object
-			if "suffix" is provided it will be appended as the second and top
-			level reverse domain.
-			if omitted it is implicitely set to the second and top level reverse
-			domain of the according IP address family
-		"""
-		if self.family == socket.AF_INET:
-			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
-			if not suffix:
-				suffix = "in-addr.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-
-		elif self.family == socket.AF_INET6:
-			reversed_ip = ".".join(reversed(self.hexdump()))
-			if not suffix:
-				suffix =  "ip6.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-			
-		else:
-			return ""
-
-	def isIPv4(self):
-		""" return true if the IP object is of address family AF_INET
-		"""
-		return True if self.family == socket.AF_INET else False
-
-	def isIPv6(self):
-		""" return true if the IP object is of address family AF_INET6
-		"""
-		return True if self.family == socket.AF_INET6 else False
-
-	def getRaw(self):
-		""" returns the raw attribute - should only be set
-			to a non-empty string if prior address conversion
-			wasn't possible
-		"""
-		return self.raw
-
-	def isValidIP(self):
-		""" returns true if the IP object has been created
-			from a valid IP address or false if not
-		"""
-		return self.valid
-
-	
-	def isInNet(self, net):
-		""" returns true if the IP object is in the provided
-			network (object)
-		"""
-		# if it isn't a valid IP address, try DNS resolution
-		if not net.isValidIP() and net.getRaw() != "":
-			# Check if IP in DNS
-			return self in DNSUtils.dnsToIp(net.getRaw())
-
-		if self.family != net.family:
-			return False
-
-		if self.family == socket.AF_INET:
-			mask = ~(0xFFFFFFFFL >> net.plen)
-
-		elif self.family == socket.AF_INET6:
-			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
-		else:
-			return False
-		
-		if self.addr & mask == net.addr:
-			return True
-
-		return False
-
-	@property
-	def maskplen(self):
-		plen = 0
-		if (hasattr(self, '_maskplen')):
-			return self._plen
-		maddr = self.addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
-
-		
-	@staticmethod
-	def masktoplen(maskstr):
-		""" converts mask string to prefix length
-			only used for IPv4 masks
-		"""
-		return IPAddr(maskstr).maskplen
-
-
-	@staticmethod
-	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
-		"""
-		match = IPAddr.IP_CRE.match(text)
-		if match:
-			return match
-		else:
-			match = IPAddr.IP6_CRE.match(text)
-			if match:
-				return match
-			else:
-				return None
-
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
new file mode 100644
index 00000000..e75cd9ed
--- /dev/null
+++ b/fail2ban/server/ipdns.py
@@ -0,0 +1,389 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester"
+__copyright__ = "Copyright (c) 2004-2016 Fail2ban Developers"
+__license__ = "GPL"
+
+import re
+import socket
+import struct
+
+from .utils import Utils
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger(__name__)
+
+
+##
+# Helper functions
+#
+#
+def asip(ip):
+	"""A little helper to guarantee ip being an IPAddr instance"""
+	if isinstance(ip, IPAddr):
+		return ip
+	return IPAddr(ip)
+
+
+##
+# Utils class for DNS handling.
+#
+# This class contains only static methods used to handle DNS 
+#
+class DNSUtils:
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	@staticmethod
+	def dnsToIp(dns):
+		""" Convert a DNS into an IP address using the Python socket module.
+			Thanks to Kevin Drapel.
+		"""
+		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
+		ips = DNSUtils.CACHE_nameToIp.get(dns)
+		if ips is not None: 
+			return ips
+		# retrieve ips
+		try:
+			ips = list()
+			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
+				ip = IPAddr(result[4][0])
+				if ip.isValidIP():
+					ips.append(ip)
+		except socket.error, e:
+			# todo: make configurable the expired time of cache entry:
+			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
+			ips = list()
+		DNSUtils.CACHE_nameToIp.set(dns, ips)
+		return ips
+
+	@staticmethod
+	def ipToName(ip):
+		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
+		v = DNSUtils.CACHE_ipToName.get(ip, ())
+		if v != ():
+			return v
+		# retrieve name
+		try:
+			if not isinstance(ip, IPAddr):
+				v = socket.gethostbyaddr(ip)[0]
+			else:
+				v = socket.gethostbyaddr(ip.ntoa())[0]
+		except socket.error, e:
+			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
+			v = None
+		DNSUtils.CACHE_ipToName.set(ip, v)
+		return v
+
+	@staticmethod
+	def textToIp(text, useDns):
+		""" Return the IP of DNS found in a given text.
+		"""
+		ipList = list()
+		# Search for plain IP
+		plainIP = IPAddr.searchIP(text)
+		if plainIP is not None:
+			ip = IPAddr(plainIP.group(0))
+			if ip.isValidIP():
+				ipList.append(ip)
+
+		# If we are allowed to resolve -- give it a try if nothing was found
+		if useDns in ("yes", "warn") and not ipList:
+			# Try to get IP from possible DNS
+			ip = DNSUtils.dnsToIp(text)
+			ipList.extend(ip)
+			if ip and useDns == "warn":
+				logSys.warning("Determined IP using DNS Lookup: %s = %s",
+					text, ipList)
+
+		return ipList
+
+
+##
+# Class for IP address handling.
+#
+# This class contains methods for handling IPv4 and IPv6 addresses.
+class IPAddr(object):
+	""" provide functions to handle IPv4 and IPv6 addresses 
+	"""
+
+	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
+	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+
+	# object attributes
+	addr = 0
+	family = socket.AF_UNSPEC
+	plen = 0
+	valid = False
+	raw = ""
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
+
+	# object methods
+	def __init(self, ipstring, cidr=-1):
+		""" initialize IP object by converting IP address string
+			to binary to integer
+		"""
+		for family in [socket.AF_INET, socket.AF_INET6]:
+			try:
+				binary = socket.inet_pton(family, ipstring)
+				self.valid = True
+				break
+			except socket.error:
+				continue
+
+		if self.valid and family == socket.AF_INET:
+			# convert host to network byte order
+			self.addr, = struct.unpack("!L", binary)
+			self.family = family
+			self.plen = 32
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+		elif self.valid and family == socket.AF_INET6:
+			# convert host to network byte order
+			hi, lo = struct.unpack("!QQ", binary)
+			self.addr = (hi << 64) | lo
+			self.family = family
+			self.plen = 128
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+			# if IPv6 address is a IPv4-compatible, make instance a IPv4
+			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
+				self.addr = lo & 0xFFFFFFFFL
+				self.family = socket.AF_INET
+				self.plen = 32
+		else:
+			# string couldn't be converted neither to a IPv4 nor
+			# to a IPv6 address - retain raw input for later use
+			# (e.g. DNS resolution)
+			self.raw = ipstring
+
+	def __repr__(self):
+		return self.ntoa()
+
+	def __str__(self):
+		return self.ntoa()
+
+	def __eq__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		if not self.valid and not other.valid: return self.raw == other.raw
+		if not self.valid or not other.valid: return False
+		if self.addr != other.addr: return False
+		if self.family != other.family: return False
+		if self.plen != other.plen: return False
+		return True
+
+	def __ne__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return True
+			other = IPAddr(other)
+		if not self.valid and not other.valid: return self.raw != other.raw
+		if self.addr != other.addr: return True
+		if self.family != other.family: return True
+		if self.plen != other.plen: return True
+		return False
+
+	def __lt__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		return self.family < other.family or self.addr < other.addr
+
+	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (self, other)
+
+	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (other, self)
+
+	def __hash__(self):
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa())
+		#return hash(self.addr)^hash((self.plen<<16)|self.family)
+
+	def hexdump(self):
+		""" dump the ip address in as a hex sequence in
+			network byte order - for debug purpose
+		"""
+		if self.family == socket.AF_INET:
+			return "%08x" % self.addr
+		elif self.family == socket.AF_INET6:
+			return "%032x" % self.addr
+		else:
+			return ""
+	
+	def ntoa(self):
+		""" represent IP object as text like the depricated 
+			C pendant inet_ntoa() but address family independent
+		"""
+		if self.family == socket.AF_INET:
+			# convert network to host byte order
+			binary = struct.pack("!L", self.addr)
+		elif self.family == socket.AF_INET6:
+			# convert network to host byte order
+			hi = self.addr >> 64
+			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			binary = struct.pack("!QQ", hi, lo)
+		else:
+			return self.getRaw()
+
+		return socket.inet_ntop(self.family, binary)
+
+	def getPTR(self, suffix=""):
+		""" generates the DNS PTR string of the provided IP address object
+			if "suffix" is provided it will be appended as the second and top
+			level reverse domain.
+			if omitted it is implicitely set to the second and top level reverse
+			domain of the according IP address family
+		"""
+		if self.family == socket.AF_INET:
+			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+			if not suffix:
+				suffix = "in-addr.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+
+		elif self.family == socket.AF_INET6:
+			reversed_ip = ".".join(reversed(self.hexdump()))
+			if not suffix:
+				suffix =  "ip6.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+			
+		else:
+			return ""
+
+	def isIPv4(self):
+		""" return true if the IP object is of address family AF_INET
+		"""
+		return True if self.family == socket.AF_INET else False
+
+	def isIPv6(self):
+		""" return true if the IP object is of address family AF_INET6
+		"""
+		return True if self.family == socket.AF_INET6 else False
+
+	def getRaw(self):
+		""" returns the raw attribute - should only be set
+			to a non-empty string if prior address conversion
+			wasn't possible
+		"""
+		return self.raw
+
+	def isValidIP(self):
+		""" returns true if the IP object has been created
+			from a valid IP address or false if not
+		"""
+		return self.valid
+
+	
+	def isInNet(self, net):
+		""" returns true if the IP object is in the provided
+			network (object)
+		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValidIP() and net.getRaw() != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.getRaw())
+
+		if self.family != net.family:
+			return False
+
+		if self.family == socket.AF_INET:
+			mask = ~(0xFFFFFFFFL >> net.plen)
+
+		elif self.family == socket.AF_INET6:
+			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
+		else:
+			return False
+		
+		if self.addr & mask == net.addr:
+			return True
+
+		return False
+
+	@property
+	def maskplen(self):
+		plen = 0
+		if (hasattr(self, '_maskplen')):
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+
+		
+	@staticmethod
+	def masktoplen(maskstr):
+		""" converts mask string to prefix length
+			only used for IPv4 masks
+		"""
+		return IPAddr(maskstr).maskplen
+
+
+	@staticmethod
+	def searchIP(text):
+		""" Search if an IP address if directly available and return
+			it.
+		"""
+		match = IPAddr.IP_CRE.match(text)
+		if match:
+			return match
+		else:
+			match = IPAddr.IP6_CRE.match(text)
+			if match:
+				return match
+			else:
+				return None
+
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 3307a6c9..130de4f2 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -27,6 +27,7 @@ __license__ = "GPL"
 import sys
 
 from ..helpers import getLogger
+from .ipdns import IPAddr
 from .mytime import MyTime
 
 # Gets the instance of the logger.
@@ -74,7 +75,6 @@ class Ticket:
 	def setIP(self, value):
 		# guarantee using IPAddr instead of unicode, str for the IP
 		if isinstance(value, basestring):
-			from .filter import IPAddr
 			value = IPAddr(value)
 		self.__ip = value
 	
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 36bc87a3..6e7bf367 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -28,7 +28,7 @@ import unittest
 
 from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
-from ..server.filter import IPAddr
+from ..server.ipdns import IPAddr
 from ..server.ticket import FailTicket
 
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 371db3bc..b5a9b8d5 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -38,8 +38,9 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils, IPAddr
+from ..server.filter import Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
+from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 0f328564..756dcc4f 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -32,7 +32,7 @@ import unittest
 from StringIO import StringIO
 
 from ..helpers import getLogger
-from ..server.filter import DNSUtils
+from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 # for action_d.test_smtp :

From 4274ae89c630e478dbf5e9c6dd55ed8dc9fe5d68 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 9 Apr 2016 00:46:17 +0200
Subject: [PATCH 15/33] Quite little adjustments on tests and return value

# Conflicts:
#	fail2ban/server/ipdns.py
---
 fail2ban/server/ipdns.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index e75cd9ed..b36b5cb1 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -303,12 +303,12 @@ class IPAddr(object):
 	def isIPv4(self):
 		""" return true if the IP object is of address family AF_INET
 		"""
-		return True if self.family == socket.AF_INET else False
+		return self.family == socket.AF_INET
 
 	def isIPv6(self):
 		""" return true if the IP object is of address family AF_INET6
 		"""
-		return True if self.family == socket.AF_INET6 else False
+		return self.family == socket.AF_INET6
 
 	def getRaw(self):
 		""" returns the raw attribute - should only be set

From dbd7e347b19d5ca0506ddc8ae2f6c25d4734e711 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 23 Apr 2016 10:08:44 +0200
Subject: [PATCH 16/33] new testcase to test beautifier code

---
 fail2ban/tests/clientbeautifiertestcase.py | 111 +++++++++++++++++++++
 fail2ban/tests/utils.py                    |   5 +
 2 files changed, 116 insertions(+)
 create mode 100644 fail2ban/tests/clientbeautifiertestcase.py

diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
new file mode 100644
index 00000000..d14030ea
--- /dev/null
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -0,0 +1,111 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Alexander Koeppe"
+__copyright__ = "Copyright (c) 2016 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
+__license__ = "GPL"
+
+import unittest
+
+from ..client.beautifier import Beautifier
+from ..version import version
+from ..ipaddr import IPAddr
+
+class BeautifierTest(unittest.TestCase):
+
+	def setUp(self):
+		""" Call before every test case """
+		self.b = Beautifier()
+
+	def tearDown(self):
+		""" Call after every test case """
+
+	def testGetInputCmd(self):
+		cmd = ["test"]
+		self.b.setInputCmd(cmd)
+		self.assertEqual(self.b.getInputCmd(), cmd)
+
+	def testPing(self):
+		self.b.setInputCmd(["ping"])
+		self.assertEqual(self.b.beautify("pong"), "Server replied: pong")
+	
+	def testVersion(self):
+		self.b.setInputCmd(["version"])
+		self.assertEqual(self.b.beautify(version), version)
+
+	def testAddJail(self):
+		self.b.setInputCmd(["add"])
+		self.assertEqual(self.b.beautify("ssh"), "Added jail ssh")
+
+	def testStartJail(self):
+		self.b.setInputCmd(["start"])
+		self.assertEqual(self.b.beautify(None), "Jail started")
+
+	def testFlushLogs(self):
+		self.b.setInputCmd(["flushlogs"])
+		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
+
+	def testStopJail(self):
+		self.b.setInputCmd(["stop", "ssh"])
+		self.assertEqual(self.b.beautify(None), "Jail stopped")
+
+	def testShutdown(self):
+		self.b.setInputCmd(["stop"])
+		self.assertEqual(self.b.beautify(None), "Shutdown successful")
+
+	def testStatus(self):
+		self.b.setInputCmd(["status"])
+		response = (("Number of jails", 0), ("Jail list", ["ssh", "exim4"]))
+		output = "Status\n|- Number of jails:\t0\n`- Jail list:\tssh exim4"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["status", "ssh"])
+		response = (
+					("Filter", [
+							("Currently failed", 0),
+							("Total failed", 0),
+							("File list", "/var/log/auth.log")
+						]
+					),
+					("Actions", [
+							("Currently banned", 3),
+							("Total banned", 3),
+							("Banned IP list", [
+									IPAddr("192.168.0.1"),
+									IPAddr("::ffff:10.2.2.1"),
+									IPAddr("2001:db8::1")
+								]
+							)
+						]
+					)
+				)
+		output = """Status for the jail: ssh
+|- Filter
+|  |- Currently failed:	0
+|  |- Total failed:	0
+|  `- File list:	/var/log/auth.log
+`- Actions
+   |- Currently banned:	3
+   |- Total banned:	3
+   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"""
+
+		self.assertEqual(self.b.beautify(response), output)
+
+
+	
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 756dcc4f..4c429105 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -133,6 +133,7 @@ def gatherTests(regexps=None, opts=None):
 	# Import all the test cases here instead of a module level to
 	# avoid circular imports
 	from . import banmanagertestcase
+	from . import clientbeautifiertestcase
 	from . import clientreadertestcase
 	from . import tickettestcase
 	from . import failmanagertestcase
@@ -187,6 +188,10 @@ def gatherTests(regexps=None, opts=None):
 		tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo))
 	except ImportError: # pragma: no cover
 		pass
+	
+	# ClientBeautifier
+	tests.addTest(unittest.makeSuite(clientbeautifiertestcase.BeautifierTest))
+
 	# ClientReaders
 	tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))

From c1a54974e921206708ce033f684188c2d9b5d12e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 23 Apr 2016 20:05:27 -0400
Subject: [PATCH 17/33] RF/ENH: 1st wave of IPAddr pythonization - properties,
 logical statements, etc

# Conflicts:
#	fail2ban/server/ipdns.py
---
 MANIFEST                                   |   1 +
 fail2ban/server/banmanager.py              |   5 +-
 fail2ban/server/filter.py                  |   2 +-
 fail2ban/server/ipdns.py                   | 284 +++++++++------------
 fail2ban/tests/clientbeautifiertestcase.py |   2 -
 5 files changed, 126 insertions(+), 168 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index 3a5b4477..11f635c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -201,6 +201,7 @@ fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
 fail2ban/tests/banmanagertestcase.py
 fail2ban/tests/clientreadertestcase.py
+fail2ban/tests/clientbeautifiertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
 fail2ban/tests/config/filter.d/simple.conf
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 67ae1b71..afa70685 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -152,8 +152,9 @@ class BanManager:
 			for banData in self.__banList:
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
-				question = ip.getPTR("origin.asn.cymru.com" if ip.isIPv4()
-						else "origin6.asn.cymru.com"
+				question = ip.getPTR(
+					"origin.asn.cymru.com" if ip.isIPv4
+					else "origin6.asn.cymru.com"
 				)
 				try:
 					answers = dns.resolver.query(question, "TXT")
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 49aba907..4bbf7996 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -383,7 +383,7 @@ class Filter(JailThread):
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
-				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValidIP() else "dns"))
+				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
 				return True
 
 		if self.__ignoreCommand:
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index b36b5cb1..744a8a73 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -17,13 +17,13 @@
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester"
+__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004-2016 Fail2ban Developers"
 __license__ = "GPL"
 
-import re
 import socket
 import struct
+import re
 
 from .utils import Utils
 from ..helpers import getLogger
@@ -38,9 +38,7 @@ logSys = getLogger(__name__)
 #
 def asip(ip):
 	"""A little helper to guarantee ip being an IPAddr instance"""
-	if isinstance(ip, IPAddr):
-		return ip
-	return IPAddr(ip)
+	return ip if isinstance(ip, IPAddr) or ip is None else IPAddr(ip)
 
 
 ##
@@ -68,7 +66,7 @@ class DNSUtils:
 			ips = list()
 			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
 				ip = IPAddr(result[4][0])
-				if ip.isValidIP():
+				if ip.isValid:
 					ips.append(ip)
 		except socket.error, e:
 			# todo: make configurable the expired time of cache entry:
@@ -88,7 +86,7 @@ class DNSUtils:
 			if not isinstance(ip, IPAddr):
 				v = socket.gethostbyaddr(ip)[0]
 			else:
-				v = socket.gethostbyaddr(ip.ntoa())[0]
+				v = socket.gethostbyaddr(ip.ntoa)[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -104,7 +102,7 @@ class DNSUtils:
 		plainIP = IPAddr.searchIP(text)
 		if plainIP is not None:
 			ip = IPAddr(plainIP.group(0))
-			if ip.isValidIP():
+			if ip.isValid:
 				ipList.append(ip)
 
 		# If we are allowed to resolve -- give it a try if nothing was found
@@ -123,134 +121,132 @@ class DNSUtils:
 # Class for IP address handling.
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
-class IPAddr(object):
-	""" provide functions to handle IPv4 and IPv6 addresses 
+#
+class IPAddr:
+	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
 
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
 
 	# object attributes
-	addr = 0
-	family = socket.AF_UNSPEC
-	plen = 0
-	valid = False
-	raw = ""
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
-		ip = IPAddr.CACHE_OBJ.get(args)
-		if ip is not None:
-			return ip
-		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
-		return ip
+	_addr = 0
+	_family = socket.AF_UNSPEC
+	_plen = 0
+	_isValid = False
+	_raw = ""
 
 	# object methods
-	def __init(self, ipstring, cidr=-1):
+	def __init__(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
-				self.valid = True
-				break
 			except socket.error:
 				continue
+			else: 
+				self._isValid = True
+				break
 
-		if self.valid and family == socket.AF_INET:
+		if self.isValid and family == socket.AF_INET:
 			# convert host to network byte order
-			self.addr, = struct.unpack("!L", binary)
-			self.family = family
-			self.plen = 32
+			self._addr, = struct.unpack("!L", binary)
+			self._family = family
+			self._plen = 32
 
 			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
+			if cidr is not None and cidr >= 0:
 				mask = ~(0xFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
+				self._addr &= mask
+				self._plen = cidr
 
-		elif self.valid and family == socket.AF_INET6:
+		elif self.isValid and family == socket.AF_INET6:
 			# convert host to network byte order
 			hi, lo = struct.unpack("!QQ", binary)
-			self.addr = (hi << 64) | lo
-			self.family = family
-			self.plen = 128
+			self._addr = (hi << 64) | lo
+			self._family = family
+			self._plen = 128
 
 			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
+			if cidr is not None and cidr >= 0:
 				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
+				self._addr &= mask
+				self._plen = cidr
 
 			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
-				self.addr = lo & 0xFFFFFFFFL
-				self.family = socket.AF_INET
-				self.plen = 32
+			elif self.isInNet(_IPv6_v4COMPAT):
+				self._addr = lo & 0xFFFFFFFFL
+				self._family = socket.AF_INET
+				self._plen = 32
 		else:
 			# string couldn't be converted neither to a IPv4 nor
 			# to a IPv6 address - retain raw input for later use
 			# (e.g. DNS resolution)
-			self.raw = ipstring
+			self._raw = ipstring
 
 	def __repr__(self):
-		return self.ntoa()
+		return self.ntoa
 
 	def __str__(self):
-		return self.ntoa()
+		return self.ntoa
+	
+	@property
+	def addr(self):
+		return self._addr
+
+	@property
+	def family(self):
+		return self._family
+
+	@property
+	def plen(self):
+		return self._plen
+
+	@property
+	def raw(self):
+		"""The raw address
+
+		Should only be set to a non-empty string if prior address
+		conversion wasn't possible
+		"""
+		return self._raw
+
+	@property
+	def isValid(self):
+		"""Either the object corresponds to a valid IP address
+		"""
+		return self._isValid
 
 	def __eq__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw == other.raw
-		if not self.valid or not other.valid: return False
-		if self.addr != other.addr: return False
-		if self.family != other.family: return False
-		if self.plen != other.plen: return False
-		return True
+		if not (self.isValid or other.isValid):
+			return self.raw == other.raw
+		return (
+			(self.isValid and other.isValid) and
+			(self.addr == other.addr) and
+			(self.family == other.family) and
+			(self.plen == other.plen)
+		)
 
 	def __ne__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return True
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw != other.raw
-		if self.addr != other.addr: return True
-		if self.family != other.family: return True
-		if self.plen != other.plen: return True
-		return False
+		return not (self == other)
 
 	def __lt__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		# should be the same as by string (because of possible compare with string):
-		return hash(self.ntoa())
-		#return hash(self.addr)^hash((self.plen<<16)|self.family)
+		return hash(self.addr) ^ hash((self.plen << 16) | self.family)
 
+	@property
 	def hexdump(self):
-		""" dump the ip address in as a hex sequence in
-			network byte order - for debug purpose
+		"""Hex representation of the IP address (for debug purposes)
 		"""
 		if self.family == socket.AF_INET:
 			return "%08x" % self.addr
@@ -258,132 +254,94 @@ class IPAddr(object):
 			return "%032x" % self.addr
 		else:
 			return ""
-	
+
+	# TODO: could be lazily evaluated
+	@property
 	def ntoa(self):
-		""" represent IP object as text like the depricated 
-			C pendant inet_ntoa() but address family independent
+		""" represent IP object as text like the deprecated
+			C pendant inet.ntoa but address family independent
 		"""
-		if self.family == socket.AF_INET:
+		if self.isIPv4:
 			# convert network to host byte order
-			binary = struct.pack("!L", self.addr)
-		elif self.family == socket.AF_INET6:
+			binary = struct.pack("!L", self._addr)
+		elif self.isIPv6:
 			# convert network to host byte order
 			hi = self.addr >> 64
 			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
 		else:
-			return self.getRaw()
+			return self.raw
 
 		return socket.inet_ntop(self.family, binary)
 
 	def getPTR(self, suffix=""):
-		""" generates the DNS PTR string of the provided IP address object
-			if "suffix" is provided it will be appended as the second and top
+		""" return the DNS PTR string of the provided IP address object
+
+			If "suffix" is provided it will be appended as the second and top
 			level reverse domain.
-			if omitted it is implicitely set to the second and top level reverse
+			If omitted it is implicitly set to the second and top level reverse
 			domain of the according IP address family
 		"""
-		if self.family == socket.AF_INET:
-			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+		if self.isIPv4:
+			exploded_ip = self.ntoa.split(".")
 			if not suffix:
 				suffix = "in-addr.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-
-		elif self.family == socket.AF_INET6:
-			reversed_ip = ".".join(reversed(self.hexdump()))
+		elif self.isIPv6:
+			exploded_ip = self.hexdump()
 			if not suffix:
-				suffix =  "ip6.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-			
+				suffix = "ip6.arpa."
 		else:
 			return ""
 
+		return "%s.%s" % (".".join(reversed(exploded_ip)), suffix)
+
+	@property
 	def isIPv4(self):
-		""" return true if the IP object is of address family AF_INET
+		"""Either the IP object is of address family AF_INET
 		"""
 		return self.family == socket.AF_INET
 
+	@property
 	def isIPv6(self):
-		""" return true if the IP object is of address family AF_INET6
+		"""Either the IP object is of address family AF_INET6
 		"""
 		return self.family == socket.AF_INET6
 
-	def getRaw(self):
-		""" returns the raw attribute - should only be set
-			to a non-empty string if prior address conversion
-			wasn't possible
-		"""
-		return self.raw
-
-	def isValidIP(self):
-		""" returns true if the IP object has been created
-			from a valid IP address or false if not
-		"""
-		return self.valid
-
-	
 	def isInNet(self, net):
-		""" returns true if the IP object is in the provided
-			network (object)
+		"""Return either the IP object is in the provided network
 		"""
-		# if it isn't a valid IP address, try DNS resolution
-		if not net.isValidIP() and net.getRaw() != "":
-			# Check if IP in DNS
-			return self in DNSUtils.dnsToIp(net.getRaw())
-
 		if self.family != net.family:
 			return False
-
-		if self.family == socket.AF_INET:
+		if self.isIPv4:
 			mask = ~(0xFFFFFFFFL >> net.plen)
-
-		elif self.family == socket.AF_INET6:
+		elif self.isIPv6:
 			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
 		else:
 			return False
 		
-		if self.addr & mask == net.addr:
-			return True
+		return self.addr & mask == net.addr
 
-		return False
-
-	@property
-	def maskplen(self):
-		plen = 0
-		if (hasattr(self, '_maskplen')):
-			return self._plen
-		maddr = self.addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
-
-		
 	@staticmethod
-	def masktoplen(maskstr):
-		""" converts mask string to prefix length
-			only used for IPv4 masks
-		"""
-		return IPAddr(maskstr).maskplen
+	def masktoplen(mask):
+		"""Convert mask string to prefix length
 
+		To be used only for IPv4 masks
+		"""
+		mask = mask.addr  # to avoid side-effect within original mask
+		plen = 0
+		while mask:
+			mask = (mask << 1) & 0xFFFFFFFFL
+			plen += 1
+		return plen
 
 	@staticmethod
 	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
+		"""Search if text is an IP address, and return it if so, else None
 		"""
 		match = IPAddr.IP_CRE.match(text)
-		if match:
-			return match
-		else:
+		if not match:
 			match = IPAddr.IP6_CRE.match(text)
-			if match:
-				return match
-			else:
-				return None
+		return match if match else None
 
+# An IPv4 compatible IPv6 to be reused
+_IPv6_v4COMPAT = IPAddr("::ffff:0:0", 96)
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index d14030ea..704aedd0 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -107,5 +107,3 @@ class BeautifierTest(unittest.TestCase):
 
 		self.assertEqual(self.b.beautify(response), output)
 
-
-	

From 9b06c325e12c49de1ef54417c428471820399341 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 10:08:20 +0200
Subject: [PATCH 18/33] 2nd wave: code review, simplification, pythonization,
 etc. (test cases passed)

---
 fail2ban/server/ipdns.py                   | 110 ++++++++++++++-------
 fail2ban/tests/clientbeautifiertestcase.py |   2 +-
 2 files changed, 77 insertions(+), 35 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 744a8a73..da820366 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -38,7 +38,9 @@ logSys = getLogger(__name__)
 #
 def asip(ip):
 	"""A little helper to guarantee ip being an IPAddr instance"""
-	return ip if isinstance(ip, IPAddr) or ip is None else IPAddr(ip)
+	if isinstance(ip, IPAddr):
+		return ip
+	return IPAddr(ip)
 
 
 ##
@@ -122,38 +124,53 @@ class DNSUtils:
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
 #
-class IPAddr:
+class IPAddr(object):
 	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
 
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+	# An IPv4 compatible IPv6 to be reused (see below)
+	IP6_4COMPAT = None
 
 	# object attributes
-	_addr = 0
-	_family = socket.AF_UNSPEC
-	_plen = 0
-	_isValid = False
-	_raw = ""
+	__slots__ = '_family','_addr','_plen','_maskplen','_raw'
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
 
 	# object methods
-	def __init__(self, ipstring, cidr=-1):
+	def __init(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
+		self._family = socket.AF_UNSPEC
+		self._addr = 0
+		self._plen = 0
+		self._maskplen = None
+		self._raw = ""
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
+				self._family = family
+				break
 			except socket.error:
 				continue
-			else: 
-				self._isValid = True
-				break
 
-		if self.isValid and family == socket.AF_INET:
+		if self._family == socket.AF_INET:
 			# convert host to network byte order
 			self._addr, = struct.unpack("!L", binary)
-			self._family = family
 			self._plen = 32
 
 			# mask out host portion if prefix length is supplied
@@ -162,11 +179,10 @@ class IPAddr:
 				self._addr &= mask
 				self._plen = cidr
 
-		elif self.isValid and family == socket.AF_INET6:
+		elif self._family == socket.AF_INET6:
 			# convert host to network byte order
 			hi, lo = struct.unpack("!QQ", binary)
 			self._addr = (hi << 64) | lo
-			self._family = family
 			self._plen = 128
 
 			# mask out host portion if prefix length is supplied
@@ -176,7 +192,7 @@ class IPAddr:
 				self._plen = cidr
 
 			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(_IPv6_v4COMPAT):
+			elif self.isInNet(IPAddr.IP6_4COMPAT):
 				self._addr = lo & 0xFFFFFFFFL
 				self._family = socket.AF_INET
 				self._plen = 32
@@ -217,32 +233,43 @@ class IPAddr:
 	def isValid(self):
 		"""Either the object corresponds to a valid IP address
 		"""
-		return self._isValid
+		return self._family != socket.AF_UNSPEC
 
 	def __eq__(self, other):
-		if not (self.isValid or other.isValid):
-			return self.raw == other.raw
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		if self._family != other._family: return False
+		if self._family == socket.AF_UNSPEC:
+			return self._raw == other._raw
 		return (
-			(self.isValid and other.isValid) and
-			(self.addr == other.addr) and
-			(self.family == other.family) and
-			(self.plen == other.plen)
+			(self._addr == other._addr) and
+			(self._plen == other._plen)
 		)
 
 	def __ne__(self, other):
 		return not (self == other)
 
 	def __lt__(self, other):
-		return self.family < other.family or self.addr < other.addr
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		return self._family < other._family or self._addr < other._addr
 
 	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		return hash(self.addr) ^ hash((self.plen << 16) | self.family)
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa)
+		#return hash(self._addr)^hash((self._plen<<16)|self._family)
 
 	@property
 	def hexdump(self):
@@ -270,7 +297,7 @@ class IPAddr:
 			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
 		else:
-			return self.raw
+			return self._raw
 
 		return socket.inet_ntop(self.family, binary)
 
@@ -310,6 +337,11 @@ class IPAddr:
 	def isInNet(self, net):
 		"""Return either the IP object is in the provided network
 		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValid and net.raw != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.raw)
+
 		if self.family != net.family:
 			return False
 		if self.isIPv4:
@@ -319,20 +351,29 @@ class IPAddr:
 		else:
 			return False
 		
-		return self.addr & mask == net.addr
+		return (self.addr & mask) == net.addr
 
+	@property
+	def maskplen(self):
+		plen = 0
+		if self._maskplen is not None:
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+		
 	@staticmethod
 	def masktoplen(mask):
 		"""Convert mask string to prefix length
 
 		To be used only for IPv4 masks
 		"""
-		mask = mask.addr  # to avoid side-effect within original mask
-		plen = 0
-		while mask:
-			mask = (mask << 1) & 0xFFFFFFFFL
-			plen += 1
-		return plen
+		return IPAddr(mask).maskplen
 
 	@staticmethod
 	def searchIP(text):
@@ -343,5 +384,6 @@ class IPAddr:
 			match = IPAddr.IP6_CRE.match(text)
 		return match if match else None
 
+
 # An IPv4 compatible IPv6 to be reused
-_IPv6_v4COMPAT = IPAddr("::ffff:0:0", 96)
+IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96)
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 704aedd0..397bce1d 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -25,7 +25,7 @@ import unittest
 
 from ..client.beautifier import Beautifier
 from ..version import version
-from ..ipaddr import IPAddr
+from ..server.ipdns import IPAddr
 
 class BeautifierTest(unittest.TestCase):
 

From 25d6cf8dd23e150055baa67d49a122712fa83379 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 19:00:06 +0200
Subject: [PATCH 19/33] fix suhosin_log in common paths - log files should be
 separated using "\n": prevents to throw an error "File option must be 'head'
 or 'tail'", if jail suhosin will be enabled.

---
 config/paths-common.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index e2f08325..9072136c 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.
 
-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s
 
 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s

From 2497b05abc30a2232c6cd846b11bc04332b9ab03 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 22:42:59 +0200
Subject: [PATCH 20/33] test cases preliminary extended;

---
 fail2ban/tests/servertestcase.py | 228 +++++++++++++++++++++++++++++++
 fail2ban/tests/utils.py          |   3 +
 2 files changed, 231 insertions(+)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d9db6c47..99afa012 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -33,7 +33,9 @@ import sys
 import platform
 
 from ..server.failregex import Regex, FailRegex, RegexException
+from ..server import actions as _actions
 from ..server.server import Server
+from ..server.ipdns import IPAddr
 from ..server.jail import Jail
 from ..server.jailthread import JailThread
 from ..server.utils import Utils
@@ -49,6 +51,8 @@ except ImportError: # pragma: no cover
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 FAST_BACKEND = "polling"
 
+logSys = getLogger("fail2ban")
+
 
 class TestServer(Server):
 	def setLogLevel(self, *args, **kwargs):
@@ -963,3 +967,227 @@ class LoggingTests(LogCaptureTestCase):
 			sys.__excepthook__ = prev_exchook
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
+
+
+from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK
+
+class ServerConfigReaderTests(LogCaptureTestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(ServerConfigReaderTests, self).__init__(*args, **kwargs)
+		self.__share_cfg = {}
+
+	def setUp(self):
+		"""Call before every test case."""
+		super(ServerConfigReaderTests, self).setUp()
+		self._execCmdLst = []
+	
+	def tearDown(self):
+		"""Call after every test case."""
+		super(ServerConfigReaderTests, self).tearDown()
+
+	def _executeCmd(self, realCmd, timeout=60):
+		for l in realCmd.split('\n'):
+			if not l.startswith('#'):
+				logSys.debug('exec-cmd: `%s`', l)
+			else:
+				logSys.debug(l)
+		return True
+
+	def test_IPAddr(self):
+		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
+		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
+
+	if STOCK:
+
+		def testCheckStockJailActions(self):
+			return
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			self.assertTrue(jails.read())		  # opens fine
+			self.assertTrue(jails.getOptions())	  # reads fine
+			stream = jails.convert(allow_no_files=True)
+
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			# for cmd in stream:
+			# 	print(cmd)
+
+			# filter all start commands (we want not start all jails):
+			for cmd in stream:
+				if cmd[0] != 'start':
+					# change to the fast init backend:
+					if cmd[0] == 'add':
+						cmd[2] = 'polling'
+					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
+					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
+						cmd[3] = "DUMMY-REGEX <HOST>"
+					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
+					try:
+						cmdHandler(cmd)
+					except Exception, e: # pragma: no cover
+						self.fail("Command %r has failed. Received %r" % (cmd, e))
+
+			# jails = server._Server__jails
+			# for j in jails:
+			# 	print(j, jails[j])
+
+		def getDefaultJailStream(self, jail, act):
+			act = act.replace('%(__name__)s', jail)
+			actName, actOpt = JailReader.extractOptions(act)
+			stream = [
+				['add', jail, 'polling'],
+				# ['set', jail, 'addfailregex', 'DUMMY-REGEX <HOST>'],
+			]
+			action = ActionReader(
+				actName, jail, actOpt,
+				share_config=self.__share_cfg, basedir=CONFIG_DIR)
+			self.assertTrue(action.read())
+			action.getOptions({})
+			stream.extend(action.convert())
+			return stream
+
+		def _assertLoggedAllTests(self, tests):
+			for t in tests:
+				self.assertLogged(t)
+
+
+		def testCheckStockCommandActions(self):
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			testJailsActions = (
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-mp`",
+						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -N f2b-j-w-iptables-mp`",
+						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`iptables -w -F f2b-j-w-iptables-mp`",
+						"`iptables -w -X f2b-j-w-iptables-mp`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -F f2b-j-w-iptables-mp`",
+						"`ip6tables -w -X f2b-j-w-iptables-mp`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-mp 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-mp -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-mp 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-mp -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-ap`",
+						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -N f2b-j-w-iptables-ap`",
+						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`iptables -w -F f2b-j-w-iptables-ap`",
+						"`iptables -w -X f2b-j-w-iptables-ap`",
+						"`ip6tables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -F f2b-j-w-iptables-ap`",
+						"`ip6tables -w -X f2b-j-w-iptables-ap`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-ap 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-ap -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-ap 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+			)
+
+			for jail, act, tests in testJailsActions:
+				stream = self.getDefaultJailStream(jail, act)
+
+				# for cmd in stream:
+				# 	print(cmd)
+
+				# filter all start commands (we want not start all jails):
+				for cmd in stream:
+					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
+					try:
+						cmdHandler(cmd)
+					except Exception, e: # pragma: no cover
+						self.fail("Command %r has failed. Received %r" % (cmd, e))
+
+			jails = server._Server__jails
+
+			for jail, act, tests in testJailsActions:
+				# print(jail, jails[jail])
+				for a in jails[jail].actions:
+					action = jails[jail].actions[a]
+					logSys.debug('# ' + ('=' * 50))
+					logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+					logSys.debug('# ' + ('=' * 50))
+					self.assertTrue(isinstance(action, _actions.CommandAction))
+					# wrap default command processor:
+					action.executeCmd = self._executeCmd
+					# test start :
+					logSys.debug('# === start ==='); self.pruneLog()
+					action.start()
+					self._assertLoggedAllTests(tests['start'])
+					# test ban ip4 :
+					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('192.0.2.1')})
+					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-ban'])
+					self.assertNotLogged(tests['ip6'])
+					# test unban ip4 :
+					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('192.0.2.1')})
+					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-unban'])
+					self.assertNotLogged(tests['ip6'])
+					# test ban ip6 :
+					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('2001:DB8::')})
+					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-ban'])
+					self.assertNotLogged(tests['ip4'])
+					# test unban ip6 :
+					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('2001:DB8::')})
+					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-unban'])
+					self.assertNotLogged(tests['ip4'])
+					# test stop :
+					logSys.debug('# === stop ==='); self.pruneLog()
+					action.stop()
+					self._assertLoggedAllTests(tests['stop'])
+
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 4c429105..fe3ab783 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -174,6 +174,7 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(servertestcase.JailTests))
 	tests.addTest(unittest.makeSuite(servertestcase.RegexTests))
 	tests.addTest(unittest.makeSuite(servertestcase.LoggingTests))
+	tests.addTest(unittest.makeSuite(servertestcase.ServerConfigReaderTests))
 	tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest))
 	tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions))
 	# Ticket, BanTicket, FailTicket
@@ -356,6 +357,8 @@ class LogCaptureTestCase(unittest.TestCase):
 				return
 		raise AssertionError("All of the %r were found present in the log: %r" % (s, logged))
 
+	def pruneLog(self):
+		self._log.truncate(0)
 
 	def getLog(self):
 		return self._log.getvalue()

From 43c0f3cdc43a31cfdf42b12d148085f3bce0dea0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 10:38:33 +0200
Subject: [PATCH 21/33] test cases extended

---
 fail2ban/tests/servertestcase.py | 60 +++++++++++++++++++++++---------
 fail2ban/tests/utils.py          | 23 ++++++++----
 2 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 99afa012..bac64dd1 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1001,7 +1001,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 	if STOCK:
 
 		def testCheckStockJailActions(self):
-			return
 			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
@@ -1020,6 +1019,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# change to the fast init backend:
 					if cmd[0] == 'add':
 						cmd[2] = 'polling'
+					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
+					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
+						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
 					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
 					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
@@ -1049,11 +1051,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			stream.extend(action.convert())
 			return stream
 
-		def _assertLoggedAllTests(self, tests):
-			for t in tests:
-				self.assertLogged(t)
-
-
 		def testCheckStockCommandActions(self):
 			server = TestServer()
 			transm = server._Server__transm
@@ -1134,6 +1131,37 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ' f2b-j-w-iptables-ipset ', 'ip6': ' f2b-j-w-iptables-ipset6 ',
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset`",
+						"`ipset destroy f2b-j-w-iptables-ipset`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset6`",
+						"`ipset destroy f2b-j-w-iptables-ipset6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`",
+					),					
+				}),
 			)
 
 			for jail, act, tests in testJailsActions:
@@ -1144,11 +1172,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 				# filter all start commands (we want not start all jails):
 				for cmd in stream:
-					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
-					try:
-						cmdHandler(cmd)
-					except Exception, e: # pragma: no cover
-						self.fail("Command %r has failed. Received %r" % (cmd, e))
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
 
 			jails = server._Server__jails
 
@@ -1165,29 +1191,29 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# test start :
 					logSys.debug('# === start ==='); self.pruneLog()
 					action.start()
-					self._assertLoggedAllTests(tests['start'])
+					self.assertLogged(*tests['start'], all=True)
 					# test ban ip4 :
 					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('192.0.2.1')})
-					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-ban'])
+					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
 					self.assertNotLogged(tests['ip6'])
 					# test unban ip4 :
 					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('192.0.2.1')})
-					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-unban'])
+					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
 					self.assertNotLogged(tests['ip6'])
 					# test ban ip6 :
 					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('2001:DB8::')})
-					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-ban'])
+					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
 					self.assertNotLogged(tests['ip4'])
 					# test unban ip6 :
 					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('2001:DB8::')})
-					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-unban'])
+					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(tests['ip4'])
 					# test stop :
 					logSys.debug('# === stop ==='); self.pruneLog()
 					action.stop()
-					self._assertLoggedAllTests(tests['stop'])
+					self.assertLogged(*tests['stop'], all=True)
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index fe3ab783..52097fa5 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -325,7 +325,7 @@ class LogCaptureTestCase(unittest.TestCase):
 	def _is_logged(self, s):
 		return s in self._log.getvalue()
 
-	def assertLogged(self, *s):
+	def assertLogged(self, *s, **kwargs):
 		"""Assert that one of the strings was logged
 
 		Preferable to assertTrue(self._is_logged(..)))
@@ -335,12 +335,22 @@ class LogCaptureTestCase(unittest.TestCase):
 		----------
 		s : string or list/set/tuple of strings
 		  Test should succeed if string (or any of the listed) is present in the log
+		all : boolean, should find all in s
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ in logged:
-				return
-		raise AssertionError("None among %r was found in the log: %r" % (s, logged))
+		if not kwargs.get('all', False):
+			# at least one entry should be found:
+			for s_ in s:
+				if s_ in logged:
+					return
+			# pragma: no cover
+			self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
+		else:
+			# each entry should be found:
+			for s_ in s:
+				if s_ not in logged:
+					# pragma: no cover
+					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
 	def assertNotLogged(self, *s):
 		"""Assert that strings were not logged
@@ -355,7 +365,8 @@ class LogCaptureTestCase(unittest.TestCase):
 		for s_ in s:
 			if s_ not in logged:
 				return
-		raise AssertionError("All of the %r were found present in the log: %r" % (s, logged))
+		# pragma: no cover
+		self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
 
 	def pruneLog(self):
 		self._log.truncate(0)

From 1a6450643d2d96738500d45a1f8ebe388f9f1b9f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 17:52:46 +0200
Subject: [PATCH 22/33] partially cherry pick from branch 'multi-set', prepare
 for conditional config parameters logic: - new readers logic (group some by
 multiple parameters 'set' -> 'multi-set'; - prevent to add 'known/'
 parameters twice (by merge section etc); - test cases fixed;

# Conflicts:
#	fail2ban/client/actionreader.py
---
 fail2ban/client/actionreader.py        | 38 +++++++++----------
 fail2ban/client/configparserinc.py     |  4 +-
 fail2ban/client/configreader.py        | 52 +++++++++++++++-----------
 fail2ban/client/fail2banregex.py       |  9 ++++-
 fail2ban/client/filterreader.py        | 22 +++++------
 fail2ban/client/jailreader.py          | 36 +++++++++---------
 fail2ban/server/server.py              | 18 +++++++--
 fail2ban/server/transmitter.py         | 37 +++++++++++++-----
 fail2ban/tests/clientreadertestcase.py | 42 +++++++++++++--------
 fail2ban/tests/samplestestcase.py      | 23 ++++++++----
 fail2ban/tests/servertestcase.py       | 19 ++++++----
 fail2ban/tests/utils.py                | 27 +++++++------
 12 files changed, 196 insertions(+), 131 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index c80b230e..698360ac 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -35,13 +35,13 @@ logSys = getLogger(__name__)
 
 class ActionReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "actionstart", None],
-		["string", "actionstop", None],
-		["string", "actioncheck", None],
-		["string", "actionban", None],
-		["string", "actionunban", None],
-	]
+	_configOpts = {
+		"actionstart": ["string", None],
+		"actionstop": ["string", None],
+		"actioncheck": ["string", None],
+		"actionban": ["string", None],
+		"actionunban": ["string", None],
+	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
 		self._name = initOpts.get("actname", file_)
@@ -65,20 +65,16 @@ class ActionReader(DefinitionInitConfigReader):
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
-		head.extend(["action", self._name])
-		for opt in self._opts:
-			if opt == "actionstart":
-				stream.append(head + ["actionstart", self._opts[opt]])
-			elif opt == "actionstop":
-				stream.append(head + ["actionstop", self._opts[opt]])
-			elif opt == "actioncheck":
-				stream.append(head + ["actioncheck", self._opts[opt]])
-			elif opt == "actionban":
-				stream.append(head + ["actionban", self._opts[opt]])
-			elif opt == "actionunban":
-				stream.append(head + ["actionunban", self._opts[opt]])
+		multi = []
+		for opt, optval in self._opts.iteritems():
+			if opt in self._configOpts:
+				multi.append([opt, optval])
 		if self._initOpts:
-			for p in self._initOpts:
-				stream.append(head + [p, self._initOpts[p]])
+			for opt, optval in self._initOpts.iteritems():
+				multi.append([opt, optval])
+		if len(multi) > 1:
+			stream.append(["multi-set", self._jailName, "action", self._name, multi])
+		elif len(multi):
+			stream.append(["set", self._jailName, "action", self._name] + multi[0])
 
 		return stream
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 7bbc7886..f4975857 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -231,7 +231,7 @@ after = 1.conf
 								# save previous known values, for possible using in local interpolations later:
 								sk = {}
 								for k, v in s2.iteritems():
-									if not k.startswith('known/'):
+									if not k.startswith('known/') and k != '__name__':
 										sk['known/'+k] = v
 								s2.update(sk)
 								# merge section
@@ -256,7 +256,7 @@ after = 1.conf
 		alls = self.get_sections()
 		sk = {}
 		for k, v in options.iteritems():
-			if pref == '' or not k.startswith(pref):
+			if pref == '' or (not k.startswith(pref) and k != '__name__'):
 				sk[pref+k] = v
 		alls[section].update(sk)
 
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index c6dd1b60..37e66249 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -203,40 +203,47 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	#
 	# Read the given option in the configuration file. Default values
 	# are used...
-	# Each optionValues entry is composed of an array with:
-	# 0 -> the type of the option
-	# 1 -> the name of the option
-	# 2 -> the default value for the option
+	# Each options entry is composed of an array with:
+	#  [[type, name, default], ...]
+	# Or it is a dict:
+	#  {name: [type, default], ...}
 	
 	def getOptions(self, sec, options, pOptions=None):
 		values = dict()
-		for option in options:
-			try:
-				if option[0] == "bool":
-					v = self.getboolean(sec, option[1])
-				elif option[0] == "int":
-					v = self.getint(sec, option[1])
+		for optname in options:
+			if isinstance(options, (list,tuple)):
+				if len(optname) > 2:
+					opttype, optname, optvalue = optname
 				else:
-					v = self.get(sec, option[1])
-				if not pOptions is None and option[1] in pOptions:
+					(opttype, optname), optvalue = optname, None
+			else:
+				opttype, optvalue = options[optname]
+			try:
+				if opttype == "bool":
+					v = self.getboolean(sec, optname)
+				elif opttype == "int":
+					v = self.getint(sec, optname)
+				else:
+					v = self.get(sec, optname)
+				if not pOptions is None and optname in pOptions:
 					continue
-				values[option[1]] = v
+				values[optname] = v
 			except NoSectionError, e:
 				# No "Definition" section or wrong basedir
 				logSys.error(e)
-				values[option[1]] = option[2]
+				values[optname] = optvalue
 				# TODO: validate error handling here.
 			except NoOptionError:
-				if not option[2] is None:
+				if not optvalue is None:
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
-								% (option[1], sec, option[2]))
-					values[option[1]] = option[2]
+								% (optname, sec, optvalue))
+					values[optname] = optvalue
 				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec)
+					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
 			except ValueError:
-				logSys.warning("Wrong value for '" + option[1] + "' in '" + sec +
-							"'. Using default one: '" + repr(option[2]) + "'")
-				values[option[1]] = option[2]
+				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
+							"'. Using default one: '" + repr(optvalue) + "'")
+				values[optname] = optvalue
 		return values
 
 
@@ -286,7 +293,8 @@ class DefinitionInitConfigReader(ConfigReader):
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
 				v = self.get("Init", opt)
-				self._initOpts['known/'+opt] = v
+				if not opt.startswith('known/') and opt != '__name__':
+					self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
 	
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index f1a9c887..9ecb7229 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -291,7 +291,14 @@ class Fail2banRegex(object):
 				RegexStat(m[3])
 				for m in filter(
 					lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype,
-					readercommands)]
+					readercommands)
+			] + [
+				RegexStat(m)
+				for mm in filter(
+					lambda x: x[0] == 'multi-set' and x[2] == "add%sregex" % regextype,
+					readercommands)
+				for m in mm[3]
+			]
 			# Read out and set possible value of maxlines
 			for command in readercommands:
 				if command[2] == "maxlines":
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index 318c8c9a..8b30f914 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -37,10 +37,10 @@ logSys = getLogger(__name__)
 
 class FilterReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "ignoreregex", None],
-		["string", "failregex", ""],
-	]
+	_configOpts = {
+		"ignoreregex": ["string", None],
+		"failregex": ["string", ""],
+	}
 
 	def setFile(self, fileName):
 		self.__file = fileName
@@ -64,16 +64,16 @@ class FilterReader(DefinitionInitConfigReader):
 		if not len(opts):
 			return stream
 		for opt, value in opts.iteritems():
-			if opt == "failregex":
+			if opt in ("failregex", "ignoreregex"):
+				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self._jailName, "addfailregex", regex])
-			elif opt == "ignoreregex":
-				for regex in value.split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self._jailName, "addignoreregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self._jailName, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self._jailName, "add" + opt, multi[0]])
 		if self._initOpts:
 			if 'maxlines' in self._initOpts:
 				# We warn when multiline regex is used without maxlines > 1
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 327ddf1b..fda5d40c 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -190,11 +190,11 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
-		for opt in self.__opts:
+		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					self.__opts.get('backend', None) != "systemd":
 				found_files = 0
-				for path in self.__opts[opt].split("\n"):
+				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
 					path, tail = path if len(path) > 1 else (path[0], "head")
 					pathList = JailReader._glob(path)
@@ -208,32 +208,32 @@ class JailReader(ConfigReader):
 					raise ValueError(
 						"Have not found any log file for %s jail" % self.__name)
 			elif opt == "logencoding":
-				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
+				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
-				backend = self.__opts[opt]
+				backend = value
 			elif opt == "maxretry":
-				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
+				stream.append(["set", self.__name, "maxretry", value])
 			elif opt == "ignoreip":
-				for ip in splitcommaspace(self.__opts[opt]):
+				for ip in splitcommaspace(value):
 					stream.append(["set", self.__name, "addignoreip", ip])
 			elif opt == "findtime":
-				stream.append(["set", self.__name, "findtime", self.__opts[opt]])
+				stream.append(["set", self.__name, "findtime", value])
 			elif opt == "bantime":
-				stream.append(["set", self.__name, "bantime", self.__opts[opt]])
+				stream.append(["set", self.__name, "bantime", value])
 			elif opt == "usedns":
-				stream.append(["set", self.__name, "usedns", self.__opts[opt]])
-			elif opt == "failregex":
-				for regex in self.__opts[opt].split('\n'):
+				stream.append(["set", self.__name, "usedns", value])
+			elif opt in ("failregex", "ignoreregex"):
+				multi = []
+				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self.__name, "addfailregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self.__name, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self.__name, "add" + opt, multi[0]])
 			elif opt == "ignorecommand":
-				stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]])
-			elif opt == "ignoreregex":
-				for regex in self.__opts[opt].split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self.__name, "addignoreregex", regex])
+				stream.append(["set", self.__name, "ignorecommand", value])
 		if self.__filter:
 			stream.extend(self.__filter.convert())
 		for action in self.__actions:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 3bdfd71b..7f75c347 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -262,8 +262,13 @@ class Server:
 	def getIgnoreCommand(self, name):
 		return self.__jails[name].filter.getIgnoreCommand()
 
-	def addFailRegex(self, name, value):
-		self.__jails[name].filter.addFailRegex(value)
+	def addFailRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addFailRegex(value)
+		else:
+			flt.addFailRegex(value)
 	
 	def delFailRegex(self, name, index):
 		self.__jails[name].filter.delFailRegex(index)
@@ -271,8 +276,13 @@ class Server:
 	def getFailRegex(self, name):
 		return self.__jails[name].filter.getFailRegex()
 	
-	def addIgnoreRegex(self, name, value):
-		self.__jails[name].filter.addIgnoreRegex(value)
+	def addIgnoreRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addIgnoreRegex(value)
+		else:
+			flt.addIgnoreRegex(value)
 	
 	def delIgnoreRegex(self, name, index):
 		self.__jails[name].filter.delIgnoreRegex(index)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 4c4c32f7..29d6d189 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -99,6 +99,8 @@ class Transmitter:
 			return None
 		elif command[0] == "flushlogs":
 			return self.__server.flushLogs()
+		elif command[0] == "multi-set":
+			return self.__commandSet(command[1:], True)
 		elif command[0] == "set":
 			return self.__commandSet(command[1:])
 		elif command[0] == "get":
@@ -109,7 +111,7 @@ class Transmitter:
 			return version.version
 		raise Exception("Invalid command")
 	
-	def __commandSet(self, command):
+	def __commandSet(self, command, multiple=False):
 		name = command[0]
 		# Logging
 		if name == "loglevel":
@@ -196,7 +198,9 @@ class Transmitter:
 			return self.__server.getJournalMatch(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
-			self.__server.addFailRegex(name, value)
+			self.__server.addFailRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getFailRegex(name)
 		elif command[1] == "delfailregex":
 			value = int(command[2])
@@ -204,7 +208,9 @@ class Transmitter:
 			return self.__server.getFailRegex(name)
 		elif command[1] == "addignoreregex":
 			value = command[2]
-			self.__server.addIgnoreRegex(name, value)
+			self.__server.addIgnoreRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getIgnoreRegex(name)
 		elif command[1] == "delignoreregex":
 			value = int(command[2])
@@ -254,15 +260,26 @@ class Transmitter:
 			return None
 		elif command[1] == "action":
 			actionname = command[2]
-			actionkey = command[3]
 			action = self.__server.getAction(name, actionname)
-			if callable(getattr(action, actionkey, None)):
-				actionvalue = json.loads(command[4]) if len(command)>4 else {}
-				return getattr(action, actionkey)(**actionvalue)
+			if multiple:
+				for cmd in command[3]:
+					actionkey = cmd[0]
+					if callable(getattr(action, actionkey, None)):
+						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}
+						getattr(action, actionkey)(**actionvalue)
+					else:
+						actionvalue = cmd[1]
+						setattr(action, actionkey, actionvalue)
+				return True
 			else:
-				actionvalue = command[4]
-				setattr(action, actionkey, actionvalue)
-				return getattr(action, actionkey)
+				actionkey = command[3]
+				if callable(getattr(action, actionkey, None)):
+					actionvalue = json.loads(command[4]) if len(command)>4 else {}
+					return getattr(action, actionkey)(**actionvalue)
+				else:
+					actionvalue = command[4]
+					setattr(action, actionkey, actionvalue)
+					return getattr(action, actionkey)
 		raise Exception("Invalid command (no set action or not yet implemented)")
 	
 	def __commandGet(self, command):
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index bd734c1b..0edbc69e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -275,7 +275,15 @@ class JailReaderTest(LogCaptureTestCase):
 		# convert and get stream
 		stream = jail.convert()
 		# get action and retrieve agent from it, compare with agent saved in version:
-		act = [o for o in stream if len(o) > 4 and (o[4] == 'agent' or o[4].endswith('badips.py'))]
+		act = []
+		for cmd in stream:
+			if len(cmd) <= 4:
+				continue
+			# differentiate between set and multi-set (wrop it here to single set):
+			if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')):
+				act.append(cmd)
+			elif cmd[0] == 'multi-set':
+				act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent'])
 		useragent = 'Fail2Ban/%s' % version
 		self.assertEqual(len(act), 4)
 		self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent])
@@ -311,23 +319,21 @@ class FilterReaderTest(unittest.TestCase):
 		self.__share_cfg = {}
 
 	def testConvert(self):
-		output = [['set', 'testcase01', 'addfailregex',
+		output = [['multi-set', 'testcase01', 'addfailregex', [
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
-			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the underlying authentication mo"
-			"dule for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"dule for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the\\nunderlying authentication."
-			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"],
+			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"]],
 			['set', 'testcase01', 'addignoreregex', 
 			"^.+ john from host 192.168.1.1\\s*$"],
 			['set', 'testcase01', 'addjournalmatch',
@@ -495,9 +501,11 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertEqual(sorted(comm_commands),
 			sorted([['add', 'emptyaction', 'auto'],
 			 ['add', 'test-known-interp', 'auto'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 3 (jail.local) <HOST>'],
+			 ['multi-set', 'test-known-interp', 'addfailregex', [
+			   'failure test 1 (filter.d/test.conf) <HOST>',
+			   'failure test 2 (filter.d/test.local) <HOST>',
+			   'failure test 3 (jail.local) <HOST>'
+			 ]],
 			 ['start', 'test-known-interp'],
 			 ['add', 'missinglogfiles', 'auto'],
 			 ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
@@ -660,12 +668,16 @@ class JailsReaderTest(LogCaptureTestCase):
 						self.assertTrue('blocktype' in action._initOpts)
 						# Verify that we have a call to set it up
 						blocktype_present = False
-						target_command = ['set', jail_name, 'action', action_name, 'blocktype']
+						target_command = [jail_name, 'action', action_name]
 						for command in commands:
-							if (len(command) > 5 and
-								command[:5] == target_command):
-								blocktype_present = True
-								continue
+							if (len(command) > 4 and command[0] == 'multi-set' and
+								command[1:4] == target_command):
+									blocktype_present = ('blocktype' in [cmd[0] for cmd in command[4]])
+							elif (len(command) > 5 and command[0] == 'set' and
+								command[1:4] == target_command and command[4] == 'blocktype'): # pragma: no cover - because of multi-set
+									blocktype_present = True
+							if blocktype_present:
+								break
 						self.assertTrue(
 							blocktype_present,
 							msg="Found no %s command among %s"
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 2ed77554..074ba24c 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -72,14 +72,21 @@ def testSampleRegexsFactory(name):
 		filterConf.getOptions({})
 
 		for opt in filterConf.convert():
-			if opt[2] == "addfailregex":
-				self.filter.addFailRegex(opt[3])
-			elif opt[2] == "maxlines":
-				self.filter.setMaxLines(opt[3])
-			elif opt[2] == "addignoreregex":
-				self.filter.addIgnoreRegex(opt[3])
-			elif opt[2] == "datepattern":
-				self.filter.setDatePattern(opt[3])
+			if opt[0] == 'multi-set':
+				optval = opt[3]
+			elif opt[0] == 'set':
+				optval = [opt[3]]
+			else:
+				continue
+			for optval in optval:
+				if opt[2] == "addfailregex":
+					self.filter.addFailRegex(optval)
+				elif opt[2] == "addignoreregex":
+					self.filter.addIgnoreRegex(optval)
+				elif opt[2] == "maxlines":
+					self.filter.setMaxLines(optval)
+				elif opt[2] == "datepattern":
+					self.filter.setDatePattern(optval)
 
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index bac64dd1..0e109a01 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1024,7 +1024,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
 					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
-					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
+					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and (
+						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
+					):
+						cmd[0] = "set"
 						cmd[3] = "DUMMY-REGEX <HOST>"
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
 					try:
@@ -1058,7 +1061,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 			testJailsActions = (
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
-					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
 						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
@@ -1095,7 +1098,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
-					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
 						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
@@ -1132,7 +1135,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
-					'ip4': ' f2b-j-w-iptables-ipset ', 'ip6': ' f2b-j-w-iptables-ipset6 ',
+					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
 						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
@@ -1196,22 +1199,22 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
-					self.assertNotLogged(tests['ip6'])
+					self.assertNotLogged(*tests['ip6'], all=True)
 					# test unban ip4 :
 					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
-					self.assertNotLogged(tests['ip6'])
+					self.assertNotLogged(*tests['ip6'], all=True)
 					# test ban ip6 :
 					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
-					self.assertNotLogged(tests['ip4'])
+					self.assertNotLogged(*tests['ip4'], all=True)
 					# test unban ip6 :
 					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
-					self.assertNotLogged(tests['ip4'])
+					self.assertNotLogged(*tests['ip4'], all=True)
 					# test stop :
 					logSys.debug('# === stop ==='); self.pruneLog()
 					action.stop()
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 52097fa5..e97daebf 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -335,7 +335,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		----------
 		s : string or list/set/tuple of strings
 		  Test should succeed if string (or any of the listed) is present in the log
-		all : boolean, should find all in s
+		all : boolean (default False) if True should fail if any of s not logged
 		"""
 		logged = self._log.getvalue()
 		if not kwargs.get('all', False):
@@ -343,16 +343,15 @@ class LogCaptureTestCase(unittest.TestCase):
 			for s_ in s:
 				if s_ in logged:
 					return
-			# pragma: no cover
-			self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
+			if True: # pragma: no cover
+				self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
 		else:
 			# each entry should be found:
 			for s_ in s:
-				if s_ not in logged:
-					# pragma: no cover
+				if s_ not in logged: # pragma: no cover
 					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
-	def assertNotLogged(self, *s):
+	def assertNotLogged(self, *s, **kwargs):
 		"""Assert that strings were not logged
 
 		Parameters
@@ -360,13 +359,19 @@ class LogCaptureTestCase(unittest.TestCase):
 		s : string or list/set/tuple of strings
 		  Test should succeed if the string (or at least one of the listed) is not
 		  present in the log
+		all : boolean (default False) if True should fail if any of s logged
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ not in logged:
-				return
-		# pragma: no cover
-		self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
+		if not kwargs.get('all', False):
+			for s_ in s:
+				if s_ not in logged:
+					return
+			if True: # pragma: no cover
+				self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
+		else:
+			for s_ in s:
+				if s_ in logged: # pragma: no cover
+					self.fail("%r was found in the log: ===\n%s===" % (s_, logged))
 
 	def pruneLog(self):
 		self._log.truncate(0)

From ed2f3ef77d4c0443790bd0e99c4f54f1cc7668fc Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 20:30:19 +0100
Subject: [PATCH 23/33] improve PF action and make IPv6 aware

---
 config/action.d/pf.conf | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index edcaa175..62296458 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -3,6 +3,7 @@
 # OpenBSD pf ban/unban
 #
 # Author: Nick Hilliard <nick@foobar.org>
+# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
 #
 #
 
@@ -12,23 +13,27 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-# we don't enable PF automatically, as it will be enabled elsewhere
-actionstart = 
+# we don't enable PF automatically; to enable run pfctl -e 
+# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to any port <port>" | pfctl -f-
 
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-# we don't disable PF automatically either
-actionstop = 
+# we only disable PF rules we've installed prior
+actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
+             pfctl -t <tablename>-<name> -T flush
+             pfctl -t <tablename>-<name> -T kill
 
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = 
+actioncheck = pfctl -sr | grep -q '<tablename>-<name>'
 
 
 # Option:  actionban
@@ -39,7 +44,7 @@ actioncheck =
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
+actionban = pfctl -t <tablename>-<name> -T add <ip>
 
 
 # Option:  actionunban
@@ -51,7 +56,7 @@ actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
 # Values:  CMD
 #
 # note -r option used to remove matching rule
-actionunban = /sbin/pfctl -t <tablename> -T delete <ip>/32
+actionunban = pfctl -t <tablename>-<name> -T delete <ip>
 
 [Init]
 # Option:  tablename
@@ -60,3 +65,15 @@ actionunban = /sbin/pfctl -t <tablename> -T delete <ip>/32
 #
 tablename = fail2ban
 
+# Option:  protocol
+# Notes.:  internally used by config reader for interpolations.
+# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
+#
+protocol = tcp
+
+# Option:  port
+# Notes.:  the port to block, defaults to any
+# Values:  [ STRING ]
+#
+port = any
+

From 75028585c02c9767d11c2deca440370aaef62214 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 19:26:27 +0200
Subject: [PATCH 24/33] test cases extended for verifying ipv4/ipv6, normalized
 pf-action with test case

---
 config/action.d/pf.conf          |   6 +-
 fail2ban/tests/servertestcase.py | 179 +++++++++++++++++++++++++++++--
 2 files changed, 176 insertions(+), 9 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 62296458..effd080c 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -24,7 +24,7 @@ actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f-
 # Values:  CMD
 #
 # we only disable PF rules we've installed prior
-actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
+actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
              pfctl -t <tablename>-<name> -T flush
              pfctl -t <tablename>-<name> -T kill
 
@@ -33,7 +33,7 @@ actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = pfctl -sr | grep -q '<tablename>-<name>'
+actioncheck = pfctl -sr | grep -q <tablename>-<name>
 
 
 # Option:  actionban
@@ -63,7 +63,7 @@ actionunban = pfctl -t <tablename>-<name> -T delete <ip>
 # Notes.:  The pf table name.
 # Values:  [ STRING ]
 #
-tablename = fail2ban
+tablename = f2b
 
 # Option:  protocol
 # Notes.:  internally used by config reader for interpolations.
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 0e109a01..893976be 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1055,13 +1055,16 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			return stream
 
 		def testCheckStockCommandActions(self):
-			server = TestServer()
-			transm = server._Server__transm
-			cmdHandler = transm._Transmitter__commandHandler
-
+			# test cases to check valid ipv4/ipv6 action definition, tuple with (('jail', 'action[params]', 'tests', ...)
+			# where tests is a dictionary contains:
+			#   'ip4' - should not be found (logged) on ban/unban of IPv6 (negative test),
+			#   'ip6' - should not be found (logged) on ban/unban of IPv4 (negative test),
+			#   'start', 'stop' - should be found (logged) on action start/stop,
+			#   etc.
 			testJailsActions = (
+				# iptables-multiport --
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
-					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
 						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
@@ -1097,8 +1100,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-mp -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				# iptables-allports --
 				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
-					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
 						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
@@ -1134,6 +1138,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				# iptables-ipset-proto6 --
 				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
@@ -1165,7 +1170,169 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`",
 					),					
 				}),
+				# iptables-ipset-proto6-allports --
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
+						"`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap`",
+						"`ip6tables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap6`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`",
+					),					
+				}),
+				# iptables --
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables`",
+						"`iptables -w -A f2b-j-w-iptables -j RETURN`",
+						"`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -N f2b-j-w-iptables`",
+						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`iptables -w -F f2b-j-w-iptables`",
+						"`iptables -w -X f2b-j-w-iptables`",
+						"`ip6tables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -F f2b-j-w-iptables`",
+						"`ip6tables -w -X f2b-j-w-iptables`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-new --
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-new`",
+						"`iptables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -N f2b-j-w-iptables-new`",
+						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`iptables -w -F f2b-j-w-iptables-new`",
+						"`iptables -w -X f2b-j-w-iptables-new`",
+						"`ip6tables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -F f2b-j-w-iptables-new`",
+						"`ip6tables -w -X f2b-j-w-iptables-new`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-new 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-new -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-new 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-new -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-xt_recent-echo --
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
+					'start': (
+						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'stop': (
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+						"`if [ `id -u` -eq 0 ];then iptables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'ip4-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip4-ban': (
+						r"`echo +192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip4-unban': (
+						r"`echo -192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-ban': (
+						r"`echo +2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip6-unban': (
+						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),					
+				}),
+				# pf --
+				('j-w-pf', 'pf[name=%(__name__)s]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
+ 						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf -T flush`',
+						'`pfctl -t f2b-j-w-pf -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
+				}),
 			)
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
 
 			for jail, act, tests in testJailsActions:
 				stream = self.getDefaultJailStream(jail, act)

From 504e5ba6f22e9e0bc5121787e1ef080d2ca98c83 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 18:40:07 +0200
Subject: [PATCH 25/33] actions support IPv6 now:   - introduced "conditional"
 sections, see for example `[Init?family=inet6]`;   - iptables-common and
 other iptables config(s) made IPv6 capable;   - several small code
 optimizations; * all test cases passed (py3.x compatible);

---
 config/action.d/iptables-common.conf          | 20 +++++
 .../iptables-ipset-proto6-allports.conf       | 26 ++++--
 config/action.d/iptables-ipset-proto6.conf    | 26 ++++--
 config/action.d/iptables-xt_recent-echo.conf  | 24 +++--
 fail2ban/client/configparserinc.py            | 46 ++++++----
 fail2ban/server/action.py                     | 88 +++++++++++++------
 fail2ban/tests/actiontestcase.py              | 16 ++--
 fail2ban/tests/filtertestcase.py              |  2 -
 8 files changed, 176 insertions(+), 72 deletions(-)

diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf
index 45819e3d..a3921021 100644
--- a/config/action.d/iptables-common.conf
+++ b/config/action.d/iptables-common.conf
@@ -6,6 +6,9 @@
 # used in all iptables based actions by default.
 #
 # The user can override the defaults in iptables-common.local
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -13,6 +16,7 @@ after = iptables-blocktype.local
         iptables-common.local
 # iptables-blocktype.local is obsolete
 
+
 [Init]
 
 # Option:  chain
@@ -62,3 +66,19 @@ lockingopt = -w
 # Notes.:  Actual command to be executed, including common to all calls options
 # Values:  STRING
 iptables = iptables <lockingopt>
+
+
+[Init?family=inet6]
+
+# Option:  blocktype (ipv6)
+# Note:    This is what the action does with rules. This can be any jump target
+#          as per the iptables man page (section 8). Common values are DROP
+#          REJECT, REJECT --reject-with icmp6-port-unreachable
+# Values:  STRING
+blocktype = REJECT --reject-with icmp6-port-unreachable
+
+# Option:  iptables (ipv6)
+# Notes.:  Actual command to be executed, including common to all calls options
+# Values:  STRING
+iptables = ip6tables <lockingopt>
+
diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf
index 1f1d336f..113f599e 100644
--- a/config/action.d/iptables-ipset-proto6-allports.conf
+++ b/config/action.d/iptables-ipset-proto6-allports.conf
@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules which probably won't be protocol version 6.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blockty
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6
diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf
index 3b51ef58..dee7b029 100644
--- a/config/action.d/iptables-ipset-proto6.conf
+++ b/config/action.d/iptables-ipset-proto6.conf
@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6
diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf
index d3d43f86..018d2cf6 100644
--- a/config/action.d/iptables-xt_recent-echo.conf
+++ b/config/action.d/iptables-xt_recent-echo.conf
@@ -2,7 +2,8 @@
 #
 # Author: Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
 #
-#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable
 
 [INCLUDES]
 
@@ -22,30 +23,30 @@ before = iptables-common.conf
 # iptables-persistent package).
 # 
 # Explanation of the rule below:
-#    Check if any packets coming from an IP on the f2b-<name>
+#    Check if any packets coming from an IP on the <iptname>
 #    list have been seen in the last 3600 seconds. If yes, update the
 #    timestamp for this IP and drop the packet. If not, let the packet
 #    through.
 #
-#    Fail2ban inserts blacklisted hosts into the f2b-<name> list
+#    Fail2ban inserts blacklisted hosts into the <iptname> list
 #    and removes them from the list after some time, according to its
 #    own rules. The 3600 second timeout is independent and acts as a
 #    safeguard in case the fail2ban process dies unexpectedly. The
 #    shorter of the two timeouts actually matters.
-actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = echo / > /proc/net/xt_recent/f2b-<name>
-             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstop = echo / > /proc/net/xt_recent/<iptname>
+             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = test -e /proc/net/xt_recent/f2b-<name>
+actioncheck = test -e /proc/net/xt_recent/<iptname>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -53,7 +54,7 @@ actioncheck = test -e /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
+actionban = echo +<ip> > /proc/net/xt_recent/<iptname>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -61,7 +62,12 @@ actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = echo -<ip> > /proc/net/xt_recent/f2b-<name>
+actionunban = echo -<ip> > /proc/net/xt_recent/<iptname>
 
 [Init]
 
+iptname              = f2b-<name>
+
+[Init?family=inet6]
+
+iptname = f2b-<name>6
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index f4975857..617ff8f6 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -25,6 +25,7 @@ __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko'
 __license__ = 'GPL'
 
 import os
+import re
 import sys
 from ..helpers import getLogger
 
@@ -99,6 +100,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
+
 	if sys.version_info >= (3,2):
 		# overload constructor only for fancy new Python3's
 		def __init__(self, share_config=None, *args, **kwargs):
@@ -225,21 +228,31 @@ after = 1.conf
 					# merge defaults and all sections to self:
 					alld.update(cfg.get_defaults())
 					for n, s in cfg.get_sections().iteritems():
-						if isinstance(s, dict):
-							s2 = alls.get(n)
-							if isinstance(s2, dict):
-								# save previous known values, for possible using in local interpolations later:
-								sk = {}
-								for k, v in s2.iteritems():
-									if not k.startswith('known/') and k != '__name__':
-										sk['known/'+k] = v
-								s2.update(sk)
-								# merge section
-								s2.update(s)
-							else:
-								alls[n] = s.copy()
+						curalls = alls
+						# conditional sections
+						cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+						if cond:
+							n, cond = cond.groups()
+							s = s.copy()
+							try: 
+								del(s['__name__'])
+							except KeyError:
+								pass
+							for k in s.keys():
+								v = s.pop(k)
+								s[k + cond] = v
+						s2 = alls.get(n)
+						if isinstance(s2, dict):
+							# save previous known values, for possible using in local interpolations later:
+							sk = {}
+							for k, v in s2.iteritems():
+								if not k.startswith('known/') and k != '__name__':
+									sk['known/'+k] = v
+							s2.update(sk)
+							# merge section
+							s2.update(s)
 						else:
-							alls[n] = s
+							alls[n] = s.copy()
 
 			return ret
 
@@ -254,9 +267,12 @@ after = 1.conf
 
 	def merge_section(self, section, options, pref='known/'):
 		alls = self.get_sections()
+		if pref == '':
+			alls[section].update(options)
+			return
 		sk = {}
 		for k, v in options.iteritems():
-			if pref == '' or (not k.startswith(pref) and k != '__name__'):
+			if not k.startswith(pref) and k != '__name__':
 				sk[pref+k] = v
 		alls[section].update(sk)
 
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 446ae853..cfa03403 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -32,6 +32,7 @@ import time
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .ipdns import asip
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -41,6 +42,11 @@ logSys = getLogger(__name__)
 # Create a lock for running system commands
 _cmd_lock = threading.Lock()
 
+# Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
+allowed_ipv6 = True
+
+# compiled RE for tag name (replacement name) 
+TAG_CRE = re.compile(r'<([^ <>]+)>')
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -256,14 +262,18 @@ class CommandAction(ActionBase):
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		if (self._properties and
-			not self.substituteRecursiveTags(self._properties)):
-			self._logSys.error(
-				"properties contain self referencing definitions "
-				"and cannot be resolved")
-			raise RuntimeError("Error starting action")
-		startCmd = self.replaceTag(self.actionstart, self._properties)
-		if not self.executeCmd(startCmd, self.timeout):
+		# check valid tags in properties (raises ValueError if self recursion, etc.):
+		if self._properties:
+			self.substituteRecursiveTags(self._properties)
+		# common (resp. ipv4):
+		startCmd = self.replaceTag('<actionstart>', self._properties, conditional='family=inet4')
+		res = self.executeCmd(startCmd, self.timeout)
+		# start ipv6 actions if available:
+		if allowed_ipv6:
+			startCmd6 = self.replaceTag('<actionstart>', self._properties, conditional='family=inet6')
+			if startCmd6 != startCmd:
+				res &= self.executeCmd(startCmd6, self.timeout)
+		if not res:
 			raise RuntimeError("Error starting action")
 
 	@property
@@ -289,7 +299,7 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionban, aInfo):
+		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
 	@property
@@ -315,7 +325,7 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionunban, aInfo):
+		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
 	@property
@@ -350,12 +360,19 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		stopCmd = self.replaceTag(self.actionstop, self._properties)
-		if not self.executeCmd(stopCmd, self.timeout):
+		# common (resp. ipv4):
+		stopCmd = self.replaceTag('<actionstop>', self._properties, conditional='family=inet4')
+		res = self.executeCmd(stopCmd, self.timeout)
+		# ipv6 actions if available:
+		if allowed_ipv6:
+			stopCmd6 = self.replaceTag('<actionstop>', self._properties, conditional='family=inet6')
+			if stopCmd6 != stopCmd:
+				res &= self.executeCmd(stopCmd6, self.timeout)
+		if not res:
 			raise RuntimeError("Error stopping action")
 
 	@classmethod
-	def substituteRecursiveTags(cls, tags):
+	def substituteRecursiveTags(cls, tags, conditional=''):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -374,7 +391,7 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
-		t = re.compile(r'<([^ <>]+)>')
+		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
 		while True:
 			repFlag = False
@@ -394,14 +411,21 @@ class CommandAction(ActionBase):
 					if found_tag == tag or found_tag in done:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
-						return False
-					if found_tag in cls._escapedTags or not found_tag in tags:
+						raise ValueError(
+							"properties contain self referencing definitions "
+							"and cannot be resolved, fail tag: %s value: %s" % (tag, value))
+					repl = None
+					if found_tag not in cls._escapedTags:
+						repl = tags.get(found_tag + '?' + conditional)
+						if repl is None:
+							repl = tags.get(found_tag)
+					if repl is None:
 						# Escaped or missing tags - just continue on searching after end of match
 						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
 						# constructs like <STDIN>.
 						m = t.search(value, m.end())
 						continue
-					value = value.replace('<%s>' % found_tag , tags[found_tag])
+					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
 					done.append(found_tag)
 					m = t.search(value, m.start())
@@ -443,7 +467,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo):
+	def replaceTag(cls, query, aInfo, conditional=''):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -459,7 +483,7 @@ class CommandAction(ActionBase):
 			`query` string with tags replaced.
 		"""
 		string = query
-		aInfo = cls.substituteRecursiveTags(aInfo)
+		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
 				value = str(aInfo[tag])			  # assure string
@@ -469,10 +493,10 @@ class CommandAction(ActionBase):
 					value = cls.escapeTag(value)
 				string = string.replace('<' + tag + '>', value)
 		# New line
-		string = string.replace("<br>", '\n')
+		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
 		return string
 
-	def _processCmd(self, cmd, aInfo = None):
+	def _processCmd(self, cmd, aInfo=None, conditional=''):
 		"""Executes a command with preliminary checks and substitutions.
 
 		Before executing any commands, executes the "check" command first
@@ -496,7 +520,17 @@ class CommandAction(ActionBase):
 			self._logSys.debug("Nothing to do")
 			return True
 
-		checkCmd = self.replaceTag(self.actioncheck, self._properties)
+		if conditional == '':
+			conditional = 'family=inet4'
+			if allowed_ipv6:
+				try:
+					ip = aInfo["ip"]
+					if ip and asip(ip).isIPv6:
+						conditional = 'family=inet6'
+				except KeyError:
+					pass
+
+		checkCmd = self.replaceTag('<actioncheck>', self._properties, conditional=conditional)
 		if not self.executeCmd(checkCmd, self.timeout):
 			self._logSys.error(
 				"Invariant check failed. Trying to restore a sane environment")
@@ -506,15 +540,15 @@ class CommandAction(ActionBase):
 				self._logSys.critical("Unable to restore environment")
 				return False
 
+		# Replace static fields
+		realCmd = self.replaceTag(cmd, self._properties, conditional=conditional)
+
 		# Replace tags
-		if not aInfo is None:
-			realCmd = self.replaceTag(cmd, aInfo)
+		if aInfo is not None:
+			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
 		else:
 			realCmd = cmd
 
-		# Replace static fields
-		realCmd = self.replaceTag(realCmd, self._properties)
-
 		return self.executeCmd(realCmd, self.timeout)
 
 	@staticmethod
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 1872eb1f..d0c1495a 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -54,12 +54,17 @@ class CommandActionTest(LogCaptureTestCase):
 			'xyz': "890 <ABC>",
 		}
 		# Recursion is bad
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# Unresolveable substition
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
@@ -127,6 +132,7 @@ class CommandActionTest(LogCaptureTestCase):
 				CallingMap(matches=lambda: str(10))),
 			"09 10 11")
 
+	def testReplaceNoTag(self):
 		# As tag not present, therefore callable should not be called
 		# Will raise ValueError if it is
 		self.assertEqual(
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index b5a9b8d5..44e06067 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1416,7 +1416,6 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip4[0], None)
 		self.assertTrue(ip4[0] is not None)
 		self.assertFalse(ip4[0] is None)
-		self.assertLess(None, ip4[0])
 		self.assertLess(ip4[0], ip4[1])
 		self.assertLess(ip4[1], ip4[2])
 		self.assertEqual(sorted(reversed(ip4)), ip4)
@@ -1424,7 +1423,6 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip6[0], None)
 		self.assertTrue(ip6[0] is not None)
 		self.assertFalse(ip6[0] is None)
-		self.assertLess(None, ip6[0])
 		self.assertLess(ip6[0], ip6[1])
 		self.assertLess(ip6[1], ip6[2])
 		self.assertEqual(sorted(reversed(ip6)), ip6)

From 941a2b6c82a392c94dac7419a1a92e1600b5dbde Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 20:00:24 +0200
Subject: [PATCH 26/33] clean up unnecessarily resp. directly unused action
 properties, because they are ambiguous now; implemented caching functionality
 for same substitutions inside replaceTag: very actual and extreme performance
 growth (up to 1000 times) for ban/unban because too slow
 substituteRecursiveTags by several tags and many includes, but totally
 unnecessary as long as parameters are not changing;

---
 fail2ban/server/action.py        | 182 ++++++++++++++-----------------
 fail2ban/tests/actiontestcase.py |  42 +++++++
 2 files changed, 123 insertions(+), 101 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index cfa03403..cdd4e28c 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -33,6 +33,7 @@ from abc import ABCMeta
 from collections import MutableMapping
 
 from .ipdns import asip
+from .mytime import MyTime
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -203,36 +204,40 @@ class CommandAction(ActionBase):
 
 	_escapedTags = set(('matches', 'ipmatches', 'ipjailmatches'))
 
+	timeout = 60
+	## Command executed in order to initialize the system.
+	actionstart = ''
+	## Command executed when an IP address gets banned.
+	actionban = ''
+	## Command executed when an IP address gets removed.
+	actionunban = ''
+	## Command executed in order to check requirements.
+	actioncheck = ''
+	## Command executed in order to stop the system.
+	actionstop = ''
+
 	def __init__(self, jail, name):
 		super(CommandAction, self).__init__(jail, name)
-		self.timeout = 60
-		## Command executed in order to initialize the system.
-		self.actionstart = ''
-		## Command executed when an IP address gets banned.
-		self.actionban = ''
-		## Command executed when an IP address gets removed.
-		self.actionunban = ''
-		## Command executed in order to check requirements.
-		self.actioncheck = ''
-		## Command executed in order to stop the system.
-		self.actionstop = ''
+		self.__properties = None
+		self.__substCache = {}
 		self._logSys.debug("Created %s" % self.__class__)
 
 	@classmethod
 	def __subclasshook__(cls, C):
 		return NotImplemented # Standard checks
 
-	@property
-	def timeout(self):
-		"""Time out period in seconds for execution of commands.
-		"""
-		return self._timeout
-
-	@timeout.setter
-	def timeout(self, timeout):
-		self._timeout = int(timeout)
-		self._logSys.debug("Set action %s timeout = %i" %
-			(self._name, self.timeout))
+	def __setattr__(self, name, value):
+		if not name.startswith('_') and not callable(value):
+			# special case for some pasrameters:
+			if name == 'timeout':
+				value = MyTime.str2seconds(value)
+			# parameters changed - clear properties and substitution cache:
+			self.__properties = None
+			self.__substCache.clear()
+			#self._logSys.debug("Set action %r %s = %r", self._name, name, value)
+			self._logSys.debug("  Set %s = %r", name, value)
+		# set:
+		self.__dict__[name] = value
 
 	@property
 	def _properties(self):
@@ -240,21 +245,20 @@ class CommandAction(ActionBase):
 
 		This is used to subsitute "tags" in the commands.
 		"""
-		return dict(
+		# if we have a properties - return it:
+		if self.__properties is not None:
+			return self.__properties
+		# otherwise retrieve:
+		self.__properties = dict(
 			(key, getattr(self, key))
 			for key in dir(self)
 			if not key.startswith("_") and not callable(getattr(self, key)))
+		#
+		return self.__properties
 
 	@property
-	def actionstart(self):
-		"""The command executed on start of the jail/action.
-		"""
-		return self._actionstart
-
-	@actionstart.setter
-	def actionstart(self, value):
-		self._actionstart = value
-		self._logSys.debug("Set actionstart = %s" % value)
+	def _substCache(self):
+		return self.__substCache
 
 	def start(self):
 		"""Executes the "actionstart" command.
@@ -263,29 +267,21 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		# check valid tags in properties (raises ValueError if self recursion, etc.):
-		if self._properties:
-			self.substituteRecursiveTags(self._properties)
-		# common (resp. ipv4):
-		startCmd = self.replaceTag('<actionstart>', self._properties, conditional='family=inet4')
-		res = self.executeCmd(startCmd, self.timeout)
-		# start ipv6 actions if available:
-		if allowed_ipv6:
-			startCmd6 = self.replaceTag('<actionstart>', self._properties, conditional='family=inet6')
-			if startCmd6 != startCmd:
-				res &= self.executeCmd(startCmd6, self.timeout)
-		if not res:
-			raise RuntimeError("Error starting action")
-
-	@property
-	def actionban(self):
-		"""The command used when a ban occurs.
-		"""
-		return self._actionban
-
-	@actionban.setter
-	def actionban(self, value):
-		self._actionban = value
-		self._logSys.debug("Set actionban = %s" % value)
+		try:
+			# common (resp. ipv4):
+			startCmd = self.replaceTag('<actionstart>', self._properties, 
+				conditional='family=inet4', cache=self.__substCache)
+			res = self.executeCmd(startCmd, self.timeout)
+			# start ipv6 actions if available:
+			if allowed_ipv6:
+				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
+					conditional='family=inet6', cache=self.__substCache)
+				if startCmd6 != startCmd:
+					res &= self.executeCmd(startCmd6, self.timeout)
+			if not res:
+				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
+		except ValueError, e:
+			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -302,17 +298,6 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
-	@property
-	def actionunban(self):
-		"""The command used when an unban occurs.
-		"""
-		return self._actionunban
-
-	@actionunban.setter
-	def actionunban(self, value):
-		self._actionunban = value
-		self._logSys.debug("Set actionunban = %s" % value)
-
 	def unban(self, aInfo):
 		"""Executes the "actionunban" command.
 
@@ -328,32 +313,6 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
-	@property
-	def actioncheck(self):
-		"""The command used to check the environment.
-
-		This is used prior to a ban taking place to ensure the
-		environment is appropriate. If this check fails, `stop` and
-		`start` is executed prior to the check being called again.
-		"""
-		return self._actioncheck
-
-	@actioncheck.setter
-	def actioncheck(self, value):
-		self._actioncheck = value
-		self._logSys.debug("Set actioncheck = %s" % value)
-
-	@property
-	def actionstop(self):
-		"""The command executed when the jail/actions stops.
-		"""
-		return self._actionstop
-
-	@actionstop.setter
-	def actionstop(self, value):
-		self._actionstop = value
-		self._logSys.debug("Set actionstop = %s" % value)
-
 	def stop(self):
 		"""Executes the "actionstop" command.
 
@@ -361,18 +320,20 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		# common (resp. ipv4):
-		stopCmd = self.replaceTag('<actionstop>', self._properties, conditional='family=inet4')
+		stopCmd = self.replaceTag('<actionstop>', self._properties, 
+			conditional='family=inet4', cache=self.__substCache)
 		res = self.executeCmd(stopCmd, self.timeout)
 		# ipv6 actions if available:
 		if allowed_ipv6:
-			stopCmd6 = self.replaceTag('<actionstop>', self._properties, conditional='family=inet6')
+			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
+				conditional='family=inet6', cache=self.__substCache)
 			if stopCmd6 != stopCmd:
 				res &= self.executeCmd(stopCmd6, self.timeout)
 		if not res:
 			raise RuntimeError("Error stopping action")
 
 	@classmethod
-	def substituteRecursiveTags(cls, tags, conditional=''):
+	def substituteRecursiveTags(cls, inptags, conditional=''):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -382,7 +343,7 @@ class CommandAction(ActionBase):
 
 		Parameters
 		----------
-		tags : dict
+		inptags : dict
 			Dictionary of tags(keys) and their values.
 
 		Returns
@@ -391,6 +352,8 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
+		# copy return tags dict to prevent modifying of inptags:
+		tags = inptags.copy()
 		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
 		while True:
@@ -467,7 +430,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo, conditional=''):
+	def replaceTag(cls, query, aInfo, conditional='', cache=None):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -482,18 +445,32 @@ class CommandAction(ActionBase):
 		str
 			`query` string with tags replaced.
 		"""
+		# use cache if allowed:
+		if cache is not None:
+			ckey = (query, conditional)
+			string = cache.get(ckey)
+			if string is not None:
+				return string
+		# replace:
 		string = query
 		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
-				value = str(aInfo[tag])			  # assure string
+				value = aInfo.get(tag + '?' + conditional)
+				if value is None:
+					value = aInfo.get(tag)
+				value = str(value)			  # assure string
 				if tag in cls._escapedTags:
 					# That one needs to be escaped since its content is
 					# out of our control
 					value = cls.escapeTag(value)
 				string = string.replace('<' + tag + '>', value)
-		# New line
+		# New line, space
 		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
+		# cache if properties:
+		if cache is not None:
+			cache[ckey] = string
+		#
 		return string
 
 	def _processCmd(self, cmd, aInfo=None, conditional=''):
@@ -520,6 +497,7 @@ class CommandAction(ActionBase):
 			self._logSys.debug("Nothing to do")
 			return True
 
+		# conditional corresponding family of the given ip:
 		if conditional == '':
 			conditional = 'family=inet4'
 			if allowed_ipv6:
@@ -530,7 +508,8 @@ class CommandAction(ActionBase):
 				except KeyError:
 					pass
 
-		checkCmd = self.replaceTag('<actioncheck>', self._properties, conditional=conditional)
+		checkCmd = self.replaceTag('<actioncheck>', self._properties, 
+			conditional=conditional, cache=self.__substCache)
 		if not self.executeCmd(checkCmd, self.timeout):
 			self._logSys.error(
 				"Invariant check failed. Trying to restore a sane environment")
@@ -541,7 +520,8 @@ class CommandAction(ActionBase):
 				return False
 
 		# Replace static fields
-		realCmd = self.replaceTag(cmd, self._properties, conditional=conditional)
+		realCmd = self.replaceTag(cmd, self._properties, 
+			conditional=conditional, cache=self.__substCache)
 
 		# Replace tags
 		if aInfo is not None:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index d0c1495a..4a34a305 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -139,6 +139,48 @@ class CommandActionTest(LogCaptureTestCase):
 			self.__action.replaceTag("abc",
 				CallingMap(matches=lambda: int("a"))), "abc")
 
+	def testReplaceTagConditionalCached(self):
+		setattr(self.__action, 'abc', "123")
+		setattr(self.__action, 'abc?family=inet4', "345")
+		setattr(self.__action, 'abc?family=inet6', "567")
+		setattr(self.__action, 'xyz', "890-<abc>")
+		setattr(self.__action, 'banaction', "Text <xyz> text <abc>")
+		# test replacement in sub tags and direct, conditional, cached:
+		cache = self.__action._substCache
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 890-123 text 123")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 890-345 text 345")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 890-567 text 567")
+		self.assertEqual(len(cache) if cache is not None else -1, 3)
+		# set one parameter - internal properties and cache should be reseted:
+		setattr(self.__action, 'xyz', "000-<abc>")
+		self.assertEqual(len(cache) if cache is not None else -1, 0)
+		# test againg, should have 000 instead of 890:
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 000-123 text 123")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 000-345 text 345")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 000-567 text 567")
+		self.assertEqual(len(cache), 3)
+
+
 	def testExecuteActionBan(self):
 		self.__action.actionstart = "touch /tmp/fail2ban.test"
 		self.assertEqual(self.__action.actionstart, "touch /tmp/fail2ban.test")

From 1b21f21c22fceedee7c178236c5d432c46c2bddd Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Fri, 6 May 2016 16:04:21 +0200
Subject: [PATCH 27/33] CIDR represenation if IPAddr object is a network prefix

---
 fail2ban/server/ipdns.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index da820366..cd22172a 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -203,10 +203,15 @@ class IPAddr(object):
 			self._raw = ipstring
 
 	def __repr__(self):
-		return self.ntoa
+		if self.isIPv4 and self.plen < 32:
+			return "%s/%d" % (self.ntoa, self.plen)
+		elif self.isIPv6 and self.plen < 128:
+			return "%s/%d" % (self.ntoa, self.plen)
+		else:
+			return self.ntoa
 
 	def __str__(self):
-		return self.ntoa
+		return self.__repr__()
 	
 	@property
 	def addr(self):

From d65e37e93d036dce73d18235f47ace66b5486c77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 21:46:04 +0200
Subject: [PATCH 28/33] CIDR splitting functionality moved from filter to
 IPAddr; meantime commit: code review, simplification, pythonization, etc. +
 test cases extended

---
 fail2ban/server/filter.py        | 12 ++-----
 fail2ban/server/ipdns.py         | 54 +++++++++++++++++++++-----------
 fail2ban/tests/filtertestcase.py | 11 +++++++
 3 files changed, 48 insertions(+), 29 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 4bbf7996..59d2e0d7 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -343,19 +343,11 @@ class Filter(JailThread):
 		# An empty string is always false
 		if ipstr == "":
 			return
-		s = ipstr.split('/', 1)
-		# IP address without CIDR mask
-		if len(s) == 1:
-			s.insert(1, -1) # <0 means no CIDR
-		elif "." in s[1]: # 255.255.255.0 style mask
-			s[1] = IPAddr.masktoplen(s[1])
-		s[1] = long(s[1])
-		
 		# Create IP address object
-		ip = IPAddr(s[0], s[1])
+		ip = IPAddr(ipstr)
 
 		# log and append to ignore list
-		logSys.debug("Add %r to ignore list (%r, %r)", ip, s[0], s[1])
+		logSys.debug("Add %r to ignore list (%r)", ip, ipstr)
 		self.__ignoreIpList.append(ip)
 
 	def delIgnoreIP(self, ip):
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index cd22172a..8021b1d8 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -150,8 +150,20 @@ class IPAddr(object):
 		IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
-	# object methods
-	def __init(self, ipstring, cidr=-1):
+	@staticmethod
+	def __wrap_ipstr(ipstr):
+		if "/" not in ipstr:
+			return ipstr, -1
+		s = ipstr.split('/', 1)
+		# IP address without CIDR mask
+		if len(s) > 2:
+			raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,))
+		if "." in s[1]: # 255.255.255.0 style mask
+			s[1] = IPAddr.masktoplen(s[1])
+		s[1] = long(s[1])
+		return s
+		
+	def __init(self, ipstr, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -160,9 +172,13 @@ class IPAddr(object):
 		self._plen = 0
 		self._maskplen = None
 		self._raw = ""
+
+		if cidr == -1:
+			ipstr, cidr = self.__wrap_ipstr(ipstr)
+
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
-				binary = socket.inet_pton(family, ipstring)
+				binary = socket.inet_pton(family, ipstr)
 				self._family = family
 				break
 			except socket.error:
@@ -200,18 +216,13 @@ class IPAddr(object):
 			# string couldn't be converted neither to a IPv4 nor
 			# to a IPv6 address - retain raw input for later use
 			# (e.g. DNS resolution)
-			self._raw = ipstring
+			self._raw = ipstr
 
 	def __repr__(self):
-		if self.isIPv4 and self.plen < 32:
-			return "%s/%d" % (self.ntoa, self.plen)
-		elif self.isIPv6 and self.plen < 128:
-			return "%s/%d" % (self.ntoa, self.plen)
-		else:
-			return self.ntoa
+		return self.ntoa
 
 	def __str__(self):
-		return self.__repr__()
+		return self.ntoa
 	
 	@property
 	def addr(self):
@@ -280,10 +291,10 @@ class IPAddr(object):
 	def hexdump(self):
 		"""Hex representation of the IP address (for debug purposes)
 		"""
-		if self.family == socket.AF_INET:
-			return "%08x" % self.addr
-		elif self.family == socket.AF_INET6:
-			return "%032x" % self.addr
+		if self._family == socket.AF_INET:
+			return "%08x" % self._addr
+		elif self._family == socket.AF_INET6:
+			return "%032x" % self._addr
 		else:
 			return ""
 
@@ -293,18 +304,23 @@ class IPAddr(object):
 		""" represent IP object as text like the deprecated
 			C pendant inet.ntoa but address family independent
 		"""
+		add = ''
 		if self.isIPv4:
 			# convert network to host byte order
 			binary = struct.pack("!L", self._addr)
+			if self._plen and self._plen < 32:
+				add = "/%d" % self._plen
 		elif self.isIPv6:
 			# convert network to host byte order
-			hi = self.addr >> 64
-			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			hi = self._addr >> 64
+			lo = self._addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
+			if self._plen and self._plen < 128:
+				add = "/%d" % self._plen
 		else:
 			return self._raw
-
-		return socket.inet_ntop(self.family, binary)
+		
+		return socket.inet_ntop(self._family, binary) + add
 
 	def getPTR(self, suffix=""):
 		""" return the DNS PTR string of the provided IP address object
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 44e06067..2d82ab6f 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1446,6 +1446,17 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		# compare with string direct:
 		self.assertEqual(d, d2)
 
+	def testIPAddr_CIDR(self):
+		self.assertEqual(str(IPAddr('93.184.0.1', 24)), '93.184.0.0/24')
+		self.assertEqual(str(IPAddr('192.168.1.0/255.255.255.128')), '192.168.1.0/25')
+		self.assertEqual(IPAddr('93.184.0.1', 24).ntoa, '93.184.0.0/24')
+		self.assertEqual(IPAddr('192.168.1.0/255.255.255.128').ntoa, '192.168.1.0/25')
+
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8::', 120)), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8::', 120).ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+
 	def testIPAddr_CompareDNS(self):
 		ips = IPAddr('example.com')
 		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))

From 50e5a7e53811ff60cb2546597ecb7748b1675a6a Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 8 May 2016 09:09:12 +0200
Subject: [PATCH 29/33] More beautifier tests

---
 fail2ban/tests/clientbeautifiertestcase.py | 178 +++++++++++++++++++--
 1 file changed, 164 insertions(+), 14 deletions(-)

diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 397bce1d..6fc63e6f 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -26,6 +26,7 @@ import unittest
 from ..client.beautifier import Beautifier
 from ..version import version
 from ..server.ipdns import IPAddr
+from ..exceptions import UnknownJailException, DuplicateJailException
 
 class BeautifierTest(unittest.TestCase):
 
@@ -57,10 +58,6 @@ class BeautifierTest(unittest.TestCase):
 		self.b.setInputCmd(["start"])
 		self.assertEqual(self.b.beautify(None), "Jail started")
 
-	def testFlushLogs(self):
-		self.b.setInputCmd(["flushlogs"])
-		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
-
 	def testStopJail(self):
 		self.b.setInputCmd(["stop", "ssh"])
 		self.assertEqual(self.b.beautify(None), "Jail stopped")
@@ -95,15 +92,168 @@ class BeautifierTest(unittest.TestCase):
 						]
 					)
 				)
-		output = """Status for the jail: ssh
-|- Filter
-|  |- Currently failed:	0
-|  |- Total failed:	0
-|  `- File list:	/var/log/auth.log
-`- Actions
-   |- Currently banned:	3
-   |- Total banned:	3
-   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"""
-
+		output = "Status for the jail: ssh\n"
+		output += "|- Filter\n"
+		output += "|  |- Currently failed:	0\n"
+		output += "|  |- Total failed:	0\n"
+		output += "|  `- File list:	/var/log/auth.log\n"
+		output += "`- Actions\n"
+		output += "   |- Currently banned:	3\n"
+		output += "   |- Total banned:	3\n"
+		output += "   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"
 		self.assertEqual(self.b.beautify(response), output)
 
+	def testFlushLogs(self):
+		self.b.setInputCmd(["flushlogs"])
+		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
+	
+	def testSyslogSocket(self):
+		self.b.setInputCmd(["get", "syslogsocket"])
+		output = "Current syslog socket is:\n`- auto"
+		self.assertEqual(self.b.beautify("auto"), output)
+
+	def testLogTarget(self):
+		self.b.setInputCmd(["get", "logtarget"])
+		output = "Current logging target is:\n`- /var/log/fail2ban.log"
+		self.assertEqual(self.b.beautify("/var/log/fail2ban.log"), output)
+
+	def testLogLevel(self):
+		self.b.setInputCmd(["get", "loglevel"])
+		output = "Current logging level is 'INFO'"
+		self.assertEqual(self.b.beautify("INFO"), output)
+
+	def testDbFile(self):
+		self.b.setInputCmd(["get", "dbfile"])
+		response = "/var/lib/fail2ban/fail2ban.sqlite3"
+		output = "Current database file is:\n`- " + response
+		self.assertEqual(self.b.beautify(response), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testDbPurgeAge(self):
+		self.b.setInputCmd(["get", "dbpurgeage"])
+		output = "Current database purge age is:\n`- 86400seconds"
+		self.assertEqual(self.b.beautify(86400), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testLogPath(self):
+		self.b.setInputCmd(["get", "sshd", "logpath"])
+		response = []
+		output = "No file is currently monitored"
+		self.assertEqual(self.b.beautify(response), output)
+		response = ["/var/log/auth.log"]
+		output = "Current monitored log file(s):\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "addlogpath", "/var/log/messages"])
+		response = ["/var/log/messages", "/var/log/auth.log"]
+		outputadd = "Current monitored log file(s):\n"
+		outputadd += "|- /var/log/messages\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), outputadd)
+
+		self.b.setInputCmd(["set", "sshd", "dellogpath", "/var/log/messages"])
+		response = ["/var/log/auth.log"]
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testLogEncoding(self):
+		self.b.setInputCmd(["get", "sshd", "logencoding"])
+		output = "Current log encoding is set to:\nUTF-8"
+		self.assertEqual(self.b.beautify("UTF-8"), output)
+
+	def testJournalMatch(self):
+		self.b.setInputCmd(["get", "sshd", "journalmatch"])
+		self.assertEqual(self.b.beautify([]), "No journal match filter set")
+
+		self.b.setInputCmd(["set", "sshd", "addjournalmatch"])
+		response = [["_SYSTEMD_UNIT", "sshd.service"]]
+		output = "Current match filter:\n"
+		output += "_SYSTEMD_UNIT sshd.service"
+		self.assertEqual(self.b.beautify(response), output)
+		
+		response.append(["_COMM", "sshd"])
+		output += " + _COMM sshd"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "deljournalmatch"])
+		response.remove(response[1])
+		self.assertEqual(self.b.beautify(response), output.split(" + ")[0])
+
+	def testDatePattern(self):
+		self.b.setInputCmd(["get", "sshd", "datepattern"])
+		output = "Current date pattern set to: "
+		response = (None, "Default Detectors")
+		self.assertEqual(self.b.beautify(None), 
+				output + "Not set/required")
+		self.assertEqual(self.b.beautify(response), 
+				output + "Default Detectors")
+		self.assertEqual(self.b.beautify(("test", "test")), 
+				output + "test (test)")
+
+	def testIgnoreIP(self):
+		self.b.setInputCmd(["get", "sshd", "ignoreip"])
+		output = "No IP address/network is ignored"
+		self.assertEqual(self.b.beautify([]), output)
+
+		self.b.setInputCmd(["set", "sshd", "addignoreip"])
+		response = [
+			IPAddr("127.0.0.0", 8), 
+			IPAddr("::1"), 
+			IPAddr("2001:db8::", 32), 
+			IPAddr("::ffff:10.0.2.1")
+		]
+		output = "These IP addresses/networks are ignored:\n"
+		output += "|- 127.0.0.0/8\n|- ::1\n"
+		output += "|- 2001:db8::/32\n`- 10.0.2.1"
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testFailRegex(self):
+		self.b.setInputCmd(["get", "sshd", "failregex"])
+		output = "No regular expression is defined"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The following regular expression are defined:\n"
+		output += "|- [0]: ^$\n`- [1]: .*"
+		self.assertEqual(self.b.beautify(["^$", ".*"]), output)
+
+	def testActions(self):
+		self.b.setInputCmd(["get", "sshd", "actions"])
+		output = "No actions for jail sshd"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The jail sshd has the following actions:\n"
+		output += "iptables-multiport"
+		self.assertEqual(self.b.beautify(["iptables-multiport"]), output)
+
+	def testActionProperties(self):
+		self.b.setInputCmd(["get", "sshd", "actionproperties", "iptables"])
+		output = "No properties for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following properties:"
+		output += "\nactionban, actionunban"
+		response = ("actionban", "actionunban")
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testActionMethods(self):
+		self.b.setInputCmd(["get", "sshd", "actionmethods", "iptables"])
+		output = "No methods for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following methods:\n"
+		output += "ban, unban"
+		self.assertEqual(self.b.beautify(["ban", "unban"]), output)
+	
+#	def testException(self):
+#		self.b.setInputCmd(["get", "sshd", "logpath"])
+#		self.assertRaises(self.b.beautify(1), TypeError)
+
+	def testBeautifyError(self):
+		response = UnknownJailException("sshd")
+		output = "Sorry but the jail 'sshd' does not exist"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		response = DuplicateJailException("sshd")
+		output = "The jail 'sshd' already exists"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		output = "Sorry but the command is invalid"
+		self.assertEqual(self.b.beautifyError(IndexError()), output)

From 9da7bb1068d6a27497a72fc8e1691881866777d6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 22:09:14 +0200
Subject: [PATCH 30/33] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/client/beautifier.py              | 42 +++++++++++-----------
 fail2ban/tests/clientbeautifiertestcase.py |  6 ++--
 fail2ban/tests/filtertestcase.py           |  5 +++
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 308cb035..7b27e92f 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -89,45 +89,45 @@ class Beautifier:
 				msg = "\n".join(msg)
 			elif inC[1] == "syslogsocket":
 				msg = "Current syslog socket is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1] == "logtarget":
 				msg = "Current logging target is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1:2] == ['loglevel']:
 				msg = "Current logging level is "
 				if response == 1:
-					msg = msg + "ERROR"
+					msg += "ERROR"
 				elif response == 2:
-					msg = msg + "WARN"
+					msg += "WARN"
 				elif response == 3:
-					msg = msg + "INFO"
+					msg += "INFO"
 				elif response == 4:
-					msg = msg + "DEBUG"
+					msg += "DEBUG"
 				else:
-					msg = msg + repr(response)
+					msg += repr(response)
 			elif inC[1] == "dbfile":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database file is:\n"
-					msg = msg + "`- " + response
+					msg += "`- " + response
 			elif inC[1] == "dbpurgeage":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database purge age is:\n"
-					msg = msg + "`- %iseconds" % response
+					msg += "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
 				else:
 					msg = "Current monitored log file(s):\n"
 					for path in response[:-1]:
-						msg = msg + "|- " + path + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + path + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] == "logencoding":
 				msg = "Current log encoding is set to:\n"
-				msg = msg + response
+				msg += response
 			elif inC[2] in ("journalmatch", "addjournalmatch", "deljournalmatch"):
 				if len(response) == 0:
 					msg = "No journal match filter set"
@@ -137,19 +137,19 @@ class Beautifier:
 			elif inC[2] == "datepattern":
 				msg = "Current date pattern set to: "
 				if response is None:
-					msg = msg + "Not set/required"
+					msg += "Not set/required"
 				elif response[0] is None:
-					msg = msg + "%s" % response[1]
+					msg += "%s" % response[1]
 				else:
-					msg = msg + "%s (%s)" % response
+					msg += "%s (%s)" % response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"
 				else:
 					msg = "These IP addresses/networks are ignored:\n"
 					for ip in response[:-1]:
-						msg = msg + "|- " + ip + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + ip + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] in ("failregex", "addfailregex", "delfailregex",
 							"ignoreregex", "addignoreregex", "delignoreregex"):
 				if len(response) == 0:
@@ -157,10 +157,10 @@ class Beautifier:
 				else:
 					msg = "The following regular expression are defined:\n"
 					c = 0
-					for ip in response[:-1]:
-						msg = msg + "|- [" + str(c) + "]: " + ip + "\n"
+					for l in response[:-1]:
+						msg += "|- [" + str(c) + "]: " + l + "\n"
 						c += 1
-					msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1]
+					msg += "`- [" + str(c) + "]: " + response[-1]
 			elif inC[2] == "actions":
 				if len(response) == 0:
 					msg = "No actions for jail %s" % inC[1]
@@ -187,7 +187,7 @@ class Beautifier:
 			logSys.warning("Beautifier error. Please report the error")
 			logSys.error("Beautify " + repr(response) + " with "
 				+ repr(self.__inputCmd) + " failed")
-			msg = msg + repr(response)
+			msg += repr(response)
 		return msg
 
 	def beautifyError(self, response):
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 6fc63e6f..0390a8ff 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -201,8 +201,10 @@ class BeautifierTest(unittest.TestCase):
 			IPAddr("::ffff:10.0.2.1")
 		]
 		output = "These IP addresses/networks are ignored:\n"
-		output += "|- 127.0.0.0/8\n|- ::1\n"
-		output += "|- 2001:db8::/32\n`- 10.0.2.1"
+		output += "|- 127.0.0.0/8\n"
+		output += "|- ::1\n"
+		output += "|- 2001:db8::/32\n"
+		output += "`- 10.0.2.1"
 		self.assertEqual(self.b.beautify(response), output)
 
 	def testFailRegex(self):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 2d82ab6f..e8bd2e37 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1457,6 +1457,11 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 
+	def testIPAddr_CIDR_Repr(self):
+		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
+			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]
+		)
+
 	def testIPAddr_CompareDNS(self):
 		ips = IPAddr('example.com')
 		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))

From e8203dabbac5f937fc5bfd9b3f5e33cd256eeaf8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 22:51:40 +0200
Subject: [PATCH 31/33] python 2.6 compatible test cases

---
 fail2ban/tests/filtertestcase.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index e8bd2e37..5e493f83 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1416,20 +1416,20 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip4[0], None)
 		self.assertTrue(ip4[0] is not None)
 		self.assertFalse(ip4[0] is None)
-		self.assertLess(ip4[0], ip4[1])
-		self.assertLess(ip4[1], ip4[2])
+		self.assertTrue(ip4[0] < ip4[1])
+		self.assertTrue(ip4[1] < ip4[2])
 		self.assertEqual(sorted(reversed(ip4)), ip4)
 		# ip6
 		self.assertNotEqual(ip6[0], None)
 		self.assertTrue(ip6[0] is not None)
 		self.assertFalse(ip6[0] is None)
-		self.assertLess(ip6[0], ip6[1])
-		self.assertLess(ip6[1], ip6[2])
+		self.assertTrue(ip6[0] < ip6[1])
+		self.assertTrue(ip6[1] < ip6[2])
 		self.assertEqual(sorted(reversed(ip6)), ip6)
 		# ip4 vs ip6
 		self.assertNotEqual(ip4[0], ip6[0])
-		self.assertLess(ip4[0], ip6[0])
-		self.assertLess(ip4[2], ip6[2])
+		self.assertTrue(ip4[0] < ip6[0])
+		self.assertTrue(ip4[2] < ip6[2])
 		self.assertEqual(sorted(reversed(ip4+ip6)), ip4+ip6)
 		# hashing (with string as key):
 		d={

From f47fdf89180de4dcc81ad6076fca2562a9ff2789 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 13:06:16 +0200
Subject: [PATCH 32/33] code coverage: extend conditional test with direct
 replacement inside replaceTag (besides recursive sub tags)

---
 fail2ban/tests/actiontestcase.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 4a34a305..f3778022 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -149,17 +149,17 @@ class CommandActionTest(LogCaptureTestCase):
 		cache = self.__action._substCache
 		for i in range(2):
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="", cache=cache),
-				"Text 890-123 text 123")
+				"Text 890-123 text 123 '123'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet4", cache=cache),
-				"Text 890-345 text 345")
+				"Text 890-345 text 345 '345'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
-				"Text 890-567 text 567")
+				"Text 890-567 text 567 '567'")
 		self.assertEqual(len(cache) if cache is not None else -1, 3)
 		# set one parameter - internal properties and cache should be reseted:
 		setattr(self.__action, 'xyz', "000-<abc>")
@@ -167,17 +167,17 @@ class CommandActionTest(LogCaptureTestCase):
 		# test againg, should have 000 instead of 890:
 		for i in range(2):
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="", cache=cache),
-				"Text 000-123 text 123")
+				"Text 000-123 text 123 '123'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet4", cache=cache),
-				"Text 000-345 text 345")
+				"Text 000-345 text 345 '345'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
-				"Text 000-567 text 567")
+				"Text 000-567 text 567 '567'")
 		self.assertEqual(len(cache), 3)
 
 

From 23ad50dbaab5c9826bbda3115a3ecb858f4fe459 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 15:27:25 +0200
Subject: [PATCH 33/33] correct <HOST> expression in failregex (now IPv6 will
 be really found from filter); respect standard spelling of IPv6 (with port)
 enclosed in brackets ([ipv6]:port), to prevent the necessarily changing of
 many filter definitions, comparison: #IPv4: 127.0.0.1:55555 #IPv6:
 [a:b:c:d::1]:55555 some small fixes (in test cases also) and code review and
 few optimizations;

---
 fail2ban/server/failregex.py          | 18 ++++++++-
 fail2ban/server/ipdns.py              | 53 +++++++++++++++++----------
 fail2ban/tests/files/logs/apache-auth |  4 ++
 fail2ban/tests/files/logs/sshd        |  2 +
 fail2ban/tests/servertestcase.py      |  8 ++--
 fail2ban/tests/utils.py               | 10 +++++
 6 files changed, 69 insertions(+), 26 deletions(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 75e64c46..6076acc3 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -43,8 +43,8 @@ class Regex:
 	def __init__(self, regex):
 		self._matchCache = None
 		# Perform shortcuts expansions.
-		# Replace "<HOST>" with default regular expression for host.
-		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]*\w)")
+		# Resolve "<HOST>" tag using default regular expression for host:
+		regex = Regex._resolveHostTag(regex)
 		# Replace "<SKIPLINES>" with regular expression for multiple lines.
 		regexSplit = regex.split("<SKIPLINES>")
 		regex = regexSplit[0]
@@ -61,6 +61,20 @@ class Regex:
 
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
+
+	@staticmethod
+	def _resolveHostTag(regex):
+		# Replace "<HOST>" with default regular expression for host:
+		# Other candidates (see gh-1374 for the discussion about):
+		#   differentiate: r"""(?:(?:::f{4,6}:)?(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<HOST>[\w\-.^_]*\w))"""
+		#     expected many changes in filter, failregex, etc...
+		#   simple:        r"""(?:::f{4,6}:)?(?P<host>[\w\-.^_:]*\w)"""
+		#     not good enough, if not precise expressions around <HOST>, because for example will match '1.2.3.4:23930' as ip-address;
+		# Todo: move this functionality to filter reader, as default <HOST> replacement,
+		#       make it configurable (via jail/filter configs)
+		return regex.replace("<HOST>",
+		 	r"""(?:::f{4,6}:)?(?P<host>(?:\d{1,3}\.){3}\d{1,3}|\[?(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}\]?|(?<=:):)|[\w\-.^_]*\w)""")
+
 	##
 	# Gets the regular expression.
 	#
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 8021b1d8..bca03428 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -103,7 +103,7 @@ class DNSUtils:
 		# Search for plain IP
 		plainIP = IPAddr.searchIP(text)
 		if plainIP is not None:
-			ip = IPAddr(plainIP.group(0))
+			ip = IPAddr(plainIP)
 			if ip.isValid:
 				ipList.append(ip)
 
@@ -127,9 +127,8 @@ class DNSUtils:
 class IPAddr(object):
 	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
-
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+	IP_4_6_CRE = re.compile(
+	  r"""^(?:(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?)$""")
 	# An IPv4 compatible IPv6 to be reused (see below)
 	IP6_4COMPAT = None
 
@@ -139,19 +138,33 @@ class IPAddr(object):
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
 
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
+	def __new__(cls, ipstr, cidr=-1):
+		# check already cached as IPAddr
+		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
 		if ip is not None:
 			return ip
+		# wrap mask to cidr (correct plen):
+		if cidr == -1:
+			ipstr, cidr = IPAddr.__wrap_ipstr(ipstr)
+			args = (ipstr, cidr)
+			# check cache again:
+			if cidr != -1:
+				ip = IPAddr.CACHE_OBJ.get(args)
+				if ip is not None:
+					return ip
 		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
+		ip.__init(ipstr, cidr)
 		IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
 	@staticmethod
 	def __wrap_ipstr(ipstr):
+		# because of standard spelling of IPv6 (with port) enclosed in brackets ([ipv6]:port),
+		# remove they now (be sure the <HOST> inside failregex uses this for IPv6 (has \[?...\]?)
+		if len(ipstr) > 2 and ipstr[0] == '[' and ipstr[-1] == ']':
+			ipstr = ipstr[1:-1]
+		# test mask:
 		if "/" not in ipstr:
 			return ipstr, -1
 		s = ipstr.split('/', 1)
@@ -173,9 +186,6 @@ class IPAddr(object):
 		self._maskplen = None
 		self._raw = ""
 
-		if cidr == -1:
-			ipstr, cidr = self.__wrap_ipstr(ipstr)
-
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstr)
@@ -376,17 +386,17 @@ class IPAddr(object):
 
 	@property
 	def maskplen(self):
-		plen = 0
+		mplen = 0
 		if self._maskplen is not None:
-			return self._plen
-		maddr = self.addr
+			return self._maskplen
+		maddr = self._addr
 		while maddr:
 			if not (maddr & 0x80000000):
 				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
 			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
+			mplen += 1
+		self._maskplen = mplen
+		return mplen
 		
 	@staticmethod
 	def masktoplen(mask):
@@ -400,10 +410,13 @@ class IPAddr(object):
 	def searchIP(text):
 		"""Search if text is an IP address, and return it if so, else None
 		"""
-		match = IPAddr.IP_CRE.match(text)
+		match = IPAddr.IP_4_6_CRE.match(text)
 		if not match:
-			match = IPAddr.IP6_CRE.match(text)
-		return match if match else None
+			return None
+		ipstr = match.group('IPv4')
+		if ipstr != '':
+			return ipstr
+		return match.group('IPv6')
 
 
 # An IPv4 compatible IPv6 to be reused
diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth
index 5b7b3c48..29de57eb 100644
--- a/fail2ban/tests/files/logs/apache-auth
+++ b/fail2ban/tests/files/logs/apache-auth
@@ -8,6 +8,8 @@
 
 # failJSON: { "time": "2013-07-11T01:21:43", "match": true , "host": "194.228.20.113" } 
 [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: /
+# failJSON: { "time": "2013-07-11T01:21:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Thu Jul 11 01:21:44 2013] [error] [client 2606:2800:220:1:248:1893:25c8:1946] user test-ipv6 not found: /
 
 # The failures below use the configuration described in fail2ban/tests/files/config/apache-auth
 #
@@ -56,6 +58,8 @@
 
 # failJSON: { "time": "2013-07-20T22:11:43", "match": true , "host": "127.0.0.1" } 
 [Sat Jul 20 22:11:43.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client 127.0.0.1:51548] AH01637: Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
+# failJSON: { "time": "2013-07-20T22:11:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Sat Jul 20 22:11:44.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client [2606:2800:220:1:248:1893:25c8:1946]:51548] AH01637: Authorization of user test-ipv6 to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
 
 # wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null
 # failJSON: { "time": "2013-07-20T21:42:44", "match": true , "host": "127.0.0.1" } 
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index 7baf4be7..89e4a3e2 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -3,6 +3,8 @@
 Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6
 # failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" }
 May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com
+# failJSON: { "time": "2005-05-29T20:56:56", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" }
+May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-ipv6 from 2606:2800:220:1:248:1893:25c8:1946
 
 #2
 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" }
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 893976be..1c05f5e8 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -458,9 +458,9 @@ class Transmitter(TransmitterBase):
 				"failed attempt from <HOST> again",
 			],
 			[
-				"user john at (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"failed attempt from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w) again",
+				"user john at %s" % (Regex._resolveHostTag('<HOST>')),
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
+				"failed attempt from %s again" % (Regex._resolveHostTag('<HOST>')),
 			],
 			self.jailName
 		)
@@ -483,7 +483,7 @@ class Transmitter(TransmitterBase):
 			],
 			[
 				"user john",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
 				"Dont match me!",
 			],
 			self.jailName
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index e97daebf..a0036979 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -101,6 +101,16 @@ def initTests(opts):
 		c.set('192.0.2.%s' % i, None)
 		c.set('198.51.100.%s' % i, None)
 		c.set('203.0.113.%s' % i, None)
+	if unittest.F2B.no_network: # pragma: no cover
+		# precache all wrong dns to ip's used in test cases:
+		c = DNSUtils.CACHE_nameToIp
+		for i in (
+			('999.999.999.999', []),
+			('abcdef.abcdef', []),
+			('192.168.0.', []),
+			('failed.dns.ch', []),
+		):
+			c.set(*i)
 
 
 def mtimesleep():