From a3e0cc9830969f09442fc79d8c8be6353791ec49 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 14 May 2014 16:01:35 +0200
Subject: [PATCH 001/623] introduced string to seconds (str2seconds) for
 configuration entries with time; todo: expands it for all time config
 entries;

---
 fail2ban/server/mytime.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py
index 166f4796..0a7e6a79 100644
--- a/fail2ban/server/mytime.py
+++ b/fail2ban/server/mytime.py
@@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 import datetime
+import re
 import time
 
 
@@ -96,3 +97,32 @@ class MyTime:
 			return time.localtime(x)
 		else:
 			return time.localtime(MyTime.myTime)
+
+	## 
+	# Wraps string expression like "1h 2m 3s" into number contains seconds (3723).
+	# The string expression will be evaluated as mathematical expression, spaces between each groups 
+	#   will be wrapped to "+" operand (only if any operand does not specified between).
+	# Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month
+	#   are "mo" or "mon" (like %b by date formating).
+	# Ex: 1hour+30min = 5400
+	#     0d 1h 30m   = 5400
+	#     1year-6mo   = 15778800
+	#     6 months    = 15778800
+	# warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also:
+	#       >>>> float(Test.str2seconds("1month")) / 60 / 60 / 24
+	#       30.4375
+	#       >>>> float(Test.str2seconds("1year")) / 60 / 60 / 24
+	#       365.25	
+	#
+	# @returns number (calculated seconds from expression "val")
+
+	#@staticmethod
+	def str2seconds(val):
+		for rexp, rpl in (
+			(r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), 
+			(r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60),
+		):
+			val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val)
+		val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val);
+		return eval(val)
+	str2seconds = staticmethod(str2seconds)

From 3cfdc5ecb36796ca2c5733ed8e20668821c442ee Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 Jun 2014 12:31:55 +0200
Subject: [PATCH 002/623] test cases extended; code review

---
 fail2ban/tests/dummyjail.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index 9b784f77..6be571d3 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -24,18 +24,19 @@ __license__ = "GPL"
 
 from threading import Lock
 
+from ..server.jail import Jail
 from ..server.actions import Actions
 
 
-class DummyJail(object):
+class DummyJail(Jail, object):
 	"""A simple 'jail' to suck in all the tickets generated by Filter's
 	"""
-	def __init__(self):
+	def __init__(self, backend=None):
 		self.lock = Lock()
 		self.queue = []
-		self.idle = False
-		self.database = None
-		self.actions = Actions(self)
+		super(DummyJail, self).__init__(name='DummyJail', backend=backend)
+		self.__db = None
+		self.__actions = Actions(self)
 
 	def __len__(self):
 		try:
@@ -64,3 +65,26 @@ class DummyJail(object):
 	@property
 	def name(self):
 		return "DummyJail #%s with %d tickets" % (id(self), len(self))
+
+	@property
+	def idle(self):
+		return False;
+	
+	@idle.setter
+	def idle(self, value):
+		pass
+
+	@property
+	def database(self):
+		return self.__db;
+	
+	@database.setter
+	def database(self, value):
+		self.__db = value;
+
+	@property
+	def actions(self):
+		return self.__actions;
+
+	def is_alive(self):
+		return True;

From 42199957d02de7dd2abd5c651638b635a0f7860b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 16 Sep 2014 13:50:32 +0200
Subject: [PATCH 003/623] prevent completely read of big files first time
 (after start of service), initial seek to start time using half-interval
 search algorithm (see issue #795): now polling backend only (currently not
 implemented for gamin and pyinotify backends); python3/pypy compatibility fix
 + removing obsolete code

---
 fail2ban/server/filter.py        | 90 +++++++++++++++++++++++++++++++-
 fail2ban/server/filterpoll.py    |  7 ++-
 fail2ban/tests/filtertestcase.py |  9 ++++
 3 files changed, 104 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index dae5c527..2dfeb174 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -24,6 +24,7 @@ __license__ = "GPL"
 import codecs
 import fcntl
 import locale
+import logging
 import os
 import re
 import sys
@@ -190,6 +191,7 @@ class Filter(JailThread):
 	# @param value the time
 
 	def setFindTime(self, value):
+		value = MyTime.str2seconds(value)
 		self.__findTime = value
 		self.failManager.setMaxTime(value)
 		logSys.info("Set findtime = %s" % value)
@@ -651,7 +653,7 @@ class FileFilter(Filter):
 	# MyTime.time()-self.findTime. When a failure is detected, a FailTicket
 	# is created and is added to the FailManager.
 
-	def getFailures(self, filename):
+	def getFailures(self, filename, startTime=None):
 		log = self.getLog(filename)
 		if log is None:
 			logSys.error("Unable to get failures in " + filename)
@@ -673,6 +675,11 @@ class FileFilter(Filter):
 			logSys.exception(e)
 			return False
 
+		# prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm:
+		if log.getPos() == 0 and startTime is not None:
+			# startTime = MyTime.time() - self.getFindTime()
+			self.seekToTime(log, startTime)
+
 		# yoh: has_content is just a bool, so do not expect it to
 		# change -- loop is exited upon break, and is not entered at
 		# all if upon container opening that one was empty.  If we
@@ -690,6 +697,74 @@ class FileFilter(Filter):
 			db.updateLog(self.jail, log)
 		return True
 
+	##
+	# Seeks to line with date (search using half-interval search algorithm), to start polling from it
+	#
+
+	def seekToTime(self, container, date):
+		fs = container.getFileSize()
+		if logSys.getEffectiveLevel() <= logging.DEBUG:
+			logSys.debug("Seek to find time %s (%s), file size %s", date, 
+				datetime.datetime.fromtimestamp(date).strftime("%Y-%m-%d %H:%M:%S"), fs)
+		date -= 0.009
+		minp = 0
+		maxp = fs
+		lastpos = 0
+		lastFew = 0
+		lastTime = None
+		cntr = 0
+		unixTime = None
+		lasti = 0
+		movecntr = 3
+		while maxp > minp:
+			i = int(minp + (maxp - minp) / 2)
+			pos = container.seek(i)
+			cntr += 1
+			# within next 5 lines try to find any legal datetime:
+			lncntr = 5;
+			dateTimeMatch = None
+			llen = 0
+			i = pos
+			while True:
+				line = container.readline()
+				if not line:
+					break
+				llen += len(line)
+				l = line.rstrip('\r\n')
+				timeMatch = self.dateDetector.matchTime(l)
+				if timeMatch:
+					dateTimeMatch = self.dateDetector.getTime(l[timeMatch.start():timeMatch.end()])
+				if not dateTimeMatch and lncntr:
+					lncntr -= 1
+					continue
+				break
+			# if we can't move (position not changed)
+			if i + llen == lasti:
+				movecntr -= 1
+				if movecntr <= 0:
+		 			break
+			lasti = i + llen;
+		 	# not found at this step - stop searching
+			if not dateTimeMatch:
+				break
+			unixTime = dateTimeMatch[0]
+			if unixTime >= date:
+				maxp = i
+			else:
+				minp = i + llen
+				lastFew = pos;
+				lastTime = unixTime
+			lastpos = pos
+		# if found position have a time greater as given - use smallest time we have found
+		if unixTime is None or unixTime > date:
+			unixTime = lastTime
+			lastpos = container.seek(lastFew, False)
+		else:
+			lastpos = container.seek(lastpos, False)
+		if logSys.getEffectiveLevel() <= logging.DEBUG:
+			logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, 
+				(datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr)
+		
 	def status(self, flavor="basic"):
 		"""Status of Filter plus files being monitored.
 		"""
@@ -742,6 +817,9 @@ class FileContainer:
 	def getFileName(self):
 		return self.__filename
 
+	def getFileSize(self):
+		return os.path.getsize(self.__filename);
+
 	def setEncoding(self, encoding):
 		codecs.lookup(encoding) # Raises LookupError if invalid
 		self.__encoding = encoding
@@ -788,6 +866,16 @@ class FileContainer:
 		self.__handler.seek(self.__pos)
 		return True
 
+	def seek(self, offs, endLine=True):
+		h = self.__handler
+		# seek to given position
+		h.seek(offs, 0)
+		# goto end of next line
+		if endLine:
+			h.readline()
+		# get current real position
+		return h.tell()
+
 	@staticmethod
 	def decode_line(filename, enc, line):
 		try:
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index d0b37775..0e99455e 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -57,6 +57,7 @@ class FilterPoll(FileFilter):
 		## The time of the last modification of the file.
 		self.__prevStats = dict()
 		self.__file404Cnt = dict()
+		self.__initial = dict()
 		logSys.debug("Created FilterPoll")
 
 	##
@@ -94,7 +95,11 @@ class FilterPoll(FileFilter):
 				for container in self.getLogs():
 					filename = container.getFileName()
 					if self.isModified(filename):
-						self.getFailures(filename)
+						# set start time as now - find time for first usage only (prevent performance bug with polling of big files)
+						self.getFailures(filename, 
+							(MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None
+						)
+						self.__initial[filename] = True
 						self.__modified = True
 
 				if self.__modified:
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 40879b66..c7dba5ce 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -912,6 +912,15 @@ class GetFailures(LogCaptureTestCase):
 		self.filter.getFailures(GetFailures.FILENAME_03)
 		_assert_correct_last_attempt(self, self.filter, output)
 
+	def testGetFailures03_seek(self):
+		# same test as above but with seek to 'Aug 14 11:55:04' - so other output ...
+		output = ('203.162.223.135', 5, 1124013544.0)
+
+		self.filter.addLogPath(GetFailures.FILENAME_03)
+		self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
+		self.filter.getFailures(GetFailures.FILENAME_03, output[2] - 4*60 + 1)
+		_assert_correct_last_attempt(self, self.filter, output)
+
 	def testGetFailures04(self):
 		output = [('212.41.96.186', 4, 1124013600.0),
 				  ('212.41.96.185', 4, 1124017198.0)]

From 410f9d7c101b5987071fcd829d6eeac3bba0b505 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 24 Sep 2014 13:21:37 +0200
Subject: [PATCH 004/623] filter, datedetector, datetemplate: performance
 optimizing of combination datedetector.matchTime/getTime2, because early
 getTime search a template and call template.matchTime again (so the date
 parsing was really executed twice, now just once); debug logging optimized;
 added info line log "Start Fail2ban ..." after changed logging target;

---
 fail2ban/server/action.py              | 10 +++--
 fail2ban/server/datedetector.py        | 56 +++++++++++++++++++++-----
 fail2ban/server/datetemplate.py        | 17 ++++----
 fail2ban/server/filter.py              | 16 ++++----
 fail2ban/server/server.py              |  3 +-
 fail2ban/tests/datedetectortestcase.py | 15 +++++++
 6 files changed, 89 insertions(+), 28 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index de0c8efc..85bdd166 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -592,10 +592,12 @@ class CommandAction(ActionBase):
 
 		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
 		if std_level >= logSys.getEffectiveLevel():
-			stdout.seek(0)
-			logSys.log(std_level, "%s -- stdout: %r" % (realCmd, stdout.read()))
-			stderr.seek(0)
-			logSys.log(std_level, "%s -- stderr: %r" % (realCmd, stderr.read()))
+			stdout.seek(0); msg = stdout.read()
+			if msg != '':
+				logSys.log(std_level, "%s -- stdout: %r", realCmd, msg)
+			stderr.seek(0); msg = stderr.read()
+			if msg != '':
+				logSys.log(std_level, "%s -- stderr: %r", realCmd, msg)
 		stdout.close()
 		stderr.close()
 
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index a4839f5c..dbfb9e5d 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -29,6 +29,8 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+logLevel = 6
+
 
 class DateDetector(object):
 	"""Manages one or more date templates to find a date within a log line.
@@ -147,7 +149,7 @@ class DateDetector(object):
 
 		Returns
 		-------
-		re.MatchObject
+		re.MatchObject, DateTemplate
 			The regex match returned from the first successfully matched
 			template.
 		"""
@@ -156,10 +158,11 @@ class DateDetector(object):
 			for template in self.__templates:
 				match = template.matchDate(line)
 				if not match is None:
-					logSys.debug("Matched time template %s" % template.name)
+					if logSys.getEffectiveLevel() <= logLevel:
+						logSys.log(logLevel, "Matched time template %s", template.name)
 					template.hits += 1
-					return match
-			return None
+					return (match, template)
+			return (None, None)
 		finally:
 			self.__lock.release()
 
@@ -178,7 +181,7 @@ class DateDetector(object):
 		-------
 		float
 			The Unix timestamp returned from the first successfully matched
-			template.
+			template or None if not found.
 		"""
 		self.__lock.acquire()
 		try:
@@ -187,8 +190,9 @@ class DateDetector(object):
 					date = template.getDate(line)
 					if date is None:
 						continue
-					logSys.debug("Got time %f for \"%r\" using template %s" %
-						(date[0], date[1].group(), template.name))
+					if logSys.getEffectiveLevel() <= logLevel:
+						logSys.log(logLevel, "Got time %f for \"%r\" using template %s", 
+							date[0], date[1].group(), template.name)
 					return date
 				except ValueError:
 					pass
@@ -196,6 +200,38 @@ class DateDetector(object):
 		finally:
 			self.__lock.release()
 
+	def getTime2(self, line, timeMatch = None):
+		"""Attempts to return the date on a log line using given template.
+
+		This uses the templates' `getDate` method in an attempt to find
+		a date.
+		Method 'getTime2' is a little bit faster as 'getTime' if template was specified (cause works without locking and without cycle)
+
+		Parameters
+		----------
+		line : str
+			Line which is searched by the date templates.
+		timeMatch (timeMatch, template) : (Match, DateTemplate)
+			Time match and template previously returned from matchTime
+
+		Returns
+		-------
+		float
+			The Unix timestamp returned from the first successfully matched
+			template or None if not found.
+		"""
+		date = None
+		if timeMatch:
+			template = timeMatch[1]
+			if template is not None:
+				date = template.getDate(line, timeMatch[0])
+		if date is not None:
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "Got time(2) %f for \"%r\" using template %s",
+					date[0], date[1].group(), template.name)
+			return date
+		return self.getTime(line)
+
 	def sortTemplate(self):
 		"""Sort the date templates by number of hits
 
@@ -206,9 +242,11 @@ class DateDetector(object):
 		"""
 		self.__lock.acquire()
 		try:
-			logSys.debug("Sorting the template list")
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "Sorting the template list")
 			self.__templates.sort(key=lambda x: x.hits, reverse=True)
 			t = self.__templates[0]
-			logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits))
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "Winning template: %s with %d hits", t.name, t.hits)
 		finally:
 			self.__lock.release()
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index bcd17ec1..e1f6b3de 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -98,7 +98,7 @@ class DateTemplate(object):
 		return dateMatch
 
 	@abstractmethod
-	def getDate(self, line):
+	def getDate(self, line, dateMatch=None):
 		"""Abstract method, which should return the date for a log line
 
 		This should return the date for a log line, typically taking the
@@ -134,7 +134,7 @@ class DateEpoch(DateTemplate):
 		DateTemplate.__init__(self)
 		self.regex = "(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))"
 
-	def getDate(self, line):
+	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
 
 		Parameters
@@ -148,7 +148,8 @@ class DateEpoch(DateTemplate):
 			Tuple containing a Unix timestamp, and the string of the date
 			which was matched and in turned used to calculated the timestamp.
 		"""
-		dateMatch = self.matchDate(line)
+		if not dateMatch:
+			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
 			return (float(dateMatch.group()), dateMatch)
@@ -212,7 +213,7 @@ class DatePatternRegex(DateTemplate):
 	def name(self, value):
 		raise NotImplementedError("Name derived from pattern")
 
-	def getDate(self, line):
+	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
 
 		This uses a custom version of strptime, using the named groups
@@ -229,7 +230,8 @@ class DatePatternRegex(DateTemplate):
 			Tuple containing a Unix timestamp, and the string of the date
 			which was matched and in turned used to calculated the timestamp.
 		"""
-		dateMatch = self.matchDate(line)
+		if not dateMatch:
+			dateMatch = self.matchDate(line)
 		if dateMatch:
 			groupdict = dict(
 				(key, value)
@@ -253,7 +255,7 @@ class DateTai64n(DateTemplate):
 		# yoh: we should not add an additional front anchor
 		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
 
-	def getDate(self, line):
+	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
 
 		Parameters
@@ -267,7 +269,8 @@ class DateTai64n(DateTemplate):
 			Tuple containing a Unix timestamp, and the string of the date
 			which was matched and in turned used to calculated the timestamp.
 		"""
-		dateMatch = self.matchDate(line)
+		if not dateMatch:
+			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
 			value = dateMatch.group()
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 2dfeb174..296ff101 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -406,14 +406,16 @@ class Filter(JailThread):
 			l = line.rstrip('\r\n')
 			logSys.log(7, "Working on line %r", line)
 
-			timeMatch = self.dateDetector.matchTime(l)
+			(timeMatch, template) = self.dateDetector.matchTime(l)
 			if timeMatch:
 				tupleLine  = (
 					l[:timeMatch.start()],
 					l[timeMatch.start():timeMatch.end()],
-					l[timeMatch.end():])
+					l[timeMatch.end():],
+					(timeMatch, template)
+				)
 			else:
-				tupleLine = (l, "", "")
+				tupleLine = (l, "", "", None)
 
 		return "".join(tupleLine[::2]), self.findFailure(
 			tupleLine, date, returnRawHost, checkAllRegex)
@@ -475,7 +477,7 @@ class Filter(JailThread):
 			self.__lastDate = date
 		elif timeText:
 
-			dateTimeMatch = self.dateDetector.getTime(timeText)
+			dateTimeMatch = self.dateDetector.getTime2(timeText, tupleLine[3])
 
 			if dateTimeMatch is None:
 				logSys.error("findFailure failed to parse timeText: " + timeText)
@@ -492,7 +494,7 @@ class Filter(JailThread):
 			date = self.__lastDate
 
 		self.__lineBuffer = (
-			self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:]
+			self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
 		logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer)
 
 		# Iterates over all the regular expressions.
@@ -731,9 +733,9 @@ class FileFilter(Filter):
 					break
 				llen += len(line)
 				l = line.rstrip('\r\n')
-				timeMatch = self.dateDetector.matchTime(l)
+				(timeMatch, template) = self.dateDetector.matchTime(l)
 				if timeMatch:
-					dateTimeMatch = self.dateDetector.getTime(l[timeMatch.start():timeMatch.end()])
+					dateTimeMatch = self.dateDetector.getTime2(l[timeMatch.start():timeMatch.end()], (timeMatch, template))
 				if not dateTimeMatch and lncntr:
 					lncntr -= 1
 					continue
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 3e371945..9836b61d 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -81,7 +81,7 @@ class Server:
 		self.flushLogs()
 
 	def start(self, sock, pidfile, force = False):
-		logSys.info("Starting Fail2ban v" + version.version)
+		logSys.info("Starting Fail2ban v%s", version.version)
 		
 		# Install signal handlers
 		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
@@ -443,6 +443,7 @@ class Server:
 			logger.addHandler(hdlr)
 			# Does not display this message at startup.
 			if not self.__logTarget is None:
+				logSys.info("Start Fail2ban v%s", version.version)
 				logSys.info(
 					"Changed logging target to %s for Fail2ban v%s"
 					% ((target
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index facfda6d..095b706a 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -106,6 +106,7 @@ class DateDetectorTest(unittest.TestCase):
 										 (not anchored, "bogus-prefix ")):
 				log = prefix + sdate + "[sshd] error: PAM: Authentication failure"
 
+				# with getTime:
 				logtime = self.__datedetector.getTime(log)
 				if should_match:
 					self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log))
@@ -118,6 +119,20 @@ class DateDetectorTest(unittest.TestCase):
 						self.assertEqual(logMatch.group(), sdate)
 				else:
 					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
+				# with matchTime and getTime2 (this combination used in filter) :
+				matchTime = self.__datedetector.matchTime(log)
+				logtime = self.__datedetector.getTime2(log, matchTime)
+				if should_match:
+					self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log))
+					( logUnix, logMatch ) = logtime
+					self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix))
+					if sdate.startswith('audit('):
+						# yes, special case, the group only matches the number
+						self.assertEqual(logMatch.group(), '1106513999.000')
+					else:
+						self.assertEqual(logMatch.group(), sdate)
+				else:
+					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
 
 	def testStableSortTemplate(self):
 		old_names = [x.name for x in self.__datedetector.templates]

From dad4234beb325a1d9836c525575956d768938208 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 25 Sep 2014 18:29:10 +0200
Subject: [PATCH 005/623] The tricky bug fixed - last position of log file will
 be never retrieved (#795): addJail (executed before addLog) early uses a
 "INSERT OR REPLACE" statement to update "enabled" to 1 (and add jail the
 first time used at once), but this syntax in sqlite always deletes an entry
 (cause of constraint) and inserts it again, so because of CASCADE all log
 entries with this jail will be also deleted from logs table.

---
 fail2ban/server/database.py | 6 +++++-
 fail2ban/server/filter.py   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 7de87554..fcc56f73 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -274,8 +274,12 @@ class Fail2BanDb(object):
 			Jail to be added to the database.
 		"""
 		cur.execute(
-			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
+			"INSERT OR IGNORE INTO jails(name, enabled) VALUES(?, 1)",
 			(jail.name,))
+		if cur.rowcount <= 0:
+			cur.execute(
+				"UPDATE jails SET enabled = 1 WHERE name = ? AND enabled != 1",
+				(jail.name,))
 
 	@commitandrollback
 	def delJail(self, cur, jail):
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 296ff101..ee0e2a1d 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -576,7 +576,7 @@ class FileFilter(Filter):
 				if lastpos and not tail:
 					log.setPos(lastpos)
 			self.__logs[path] = log
-			logSys.info("Added logfile = %s" % path)
+			logSys.info("Added logfile = %s (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
 			self._addLogPath(path)			# backend specific
 
 	def _addLogPath(self, path):

From d22b2498d4dd2ba3f4ea45260335783648eb62cf Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Fri, 24 Oct 2014 01:32:04 +0200
Subject: [PATCH 006/623] normalizing time config entries: use time
 abbreviation (str2seconds) for all time options such 'dbpurgeage', 'bantime',
 'findtime', ex.: default '1d' instead '86400'; code review and test case
 extended;

---
 config/fail2ban.conf                   |  2 +-
 config/jail.conf                       | 14 +++++++-------
 fail2ban/client/fail2banreader.py      |  2 +-
 fail2ban/client/jailreader.py          |  6 +++---
 fail2ban/server/database.py            |  2 +-
 fail2ban/server/mytime.py              |  6 ++++--
 fail2ban/server/transmitter.py         |  4 ++--
 fail2ban/tests/clientreadertestcase.py |  2 +-
 fail2ban/tests/databasetestcase.py     |  9 +++++++++
 fail2ban/tests/servertestcase.py       |  2 ++
 10 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/config/fail2ban.conf b/config/fail2ban.conf
index 89e2538e..7c001155 100644
--- a/config/fail2ban.conf
+++ b/config/fail2ban.conf
@@ -66,4 +66,4 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3
 # Options: dbpurgeage
 # Notes.: Sets age at which bans should be purged from the database
 # Values: [ SECONDS ] Default: 86400 (24hours)
-dbpurgeage = 86400
+dbpurgeage = 1d
diff --git a/config/jail.conf b/config/jail.conf
index c8dc6d9c..21b97798 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -18,7 +18,7 @@
 # See man 5 jail.conf for details.
 #
 # [DEFAULT]
-# bantime = 3600
+# bantime = 1h
 #
 # [sshd]
 # enabled = true
@@ -56,11 +56,11 @@ ignoreip = 127.0.0.1/8
 ignorecommand =
 
 # "bantime" is the number of seconds that a host is banned.
-bantime  = 600
+bantime  = 10m
 
 # A host is banned if it has generated "maxretry" during the last "findtime"
 # seconds.
-findtime  = 600
+findtime  = 10m
 
 # "maxretry" is the number of failures before a host get banned.
 maxretry = 5
@@ -258,7 +258,7 @@ logpath  = %(apache_error_log)s
 # for email addresses. The mail outputs are buffered.
 port     = http,https
 logpath  = %(apache_access_log)s
-bantime  = 172800
+bantime  = 48h
 maxretry = 1
 
 
@@ -738,8 +738,8 @@ maxretry = 5
 
 logpath  = /var/log/fail2ban.log
 banaction = %(banaction_allports)s
-bantime  = 604800  ; 1 week
-findtime = 86400   ; 1 day
+bantime  = 1w
+findtime = 1d
 maxretry = 5
 
 
@@ -818,7 +818,7 @@ filter       = apache-pass
 logpath      = %(apache_access_log)s
 blocktype    = RETURN
 returntype   = DROP
-bantime      = 3600
+bantime      = 1h
 maxretry     = 1
 findtime     = 1
 
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index 709f4b5d..c55f65ea 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -49,7 +49,7 @@ class Fail2banReader(ConfigReader):
 				["string", "logtarget", "STDERR"],
 				["string", "syslogsocket", "auto"],
 				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
-				["int", "dbpurgeage", 86400]]
+				["string", "dbpurgeage", "1d"]]
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
 	
 	def convert(self):
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 54ac59fa..46f910e3 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -97,9 +97,9 @@ class JailReader(ConfigReader):
 				["string", "logpath", None],
 				["string", "logencoding", None],
 				["string", "backend", "auto"],
-				["int", "maxretry", None],
-				["int", "findtime", None],
-				["int", "bantime", None],
+				["int",    "maxretry", None],
+				["string", "findtime", None],
+				["string", "bantime", None],
 				["string", "usedns", None],
 				["string", "failregex", None],
 				["string", "ignoreregex", None],
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index fcc56f73..3cfaa5a6 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -218,7 +218,7 @@ class Fail2BanDb(object):
 
 	@purgeage.setter
 	def purgeage(self, value):
-		self._purgeAge = int(value)
+		self._purgeAge = MyTime.str2seconds(value)
 
 	@commitandrollback
 	def createDb(self, cur):
diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py
index 0a7e6a79..47509f19 100644
--- a/fail2ban/server/mytime.py
+++ b/fail2ban/server/mytime.py
@@ -116,8 +116,11 @@ class MyTime:
 	#
 	# @returns number (calculated seconds from expression "val")
 
-	#@staticmethod
+	@staticmethod
 	def str2seconds(val):
+		# replace together standing abbreviations, example '1d12h' -> '1d 12h':
+		val = re.sub(r"(?i)(?<=[a-z])(\d)", r" \1", val)
+		# replace abbreviation with expression:
 		for rexp, rpl in (
 			(r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), 
 			(r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60),
@@ -125,4 +128,3 @@ class MyTime:
 			val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val)
 		val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val);
 		return eval(val)
-	str2seconds = staticmethod(str2seconds)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 0d9f0fe4..eb5b7713 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -216,7 +216,7 @@ class Transmitter:
 			return self.__server.getUseDns(name)
 		elif command[1] == "findtime":
 			value = command[2]
-			self.__server.setFindTime(name, int(value))
+			self.__server.setFindTime(name, value)
 			return self.__server.getFindTime(name)
 		elif command[1] == "datepattern":
 			value = command[2]
@@ -233,7 +233,7 @@ class Transmitter:
 		# command
 		elif command[1] == "bantime":
 			value = command[2]
-			self.__server.setBanTime(name, int(value))
+			self.__server.setBanTime(name, value)
 			return self.__server.getBanTime(name)
 		elif command[1] == "banip":
 			value = command[2]
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index d19090be..d0c37a3e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -668,7 +668,7 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertEqual(sorted(commands),
 							 [['set', 'dbfile',
 								'/var/lib/fail2ban/fail2ban.sqlite3'],
-							  ['set', 'dbpurgeage', 86400],
+							  ['set', 'dbpurgeage', '1d'],
 							  ['set', 'loglevel', "INFO"],
 							  ['set', 'logtarget', '/var/log/fail2ban.log'],
 							  ['set', 'syslogsocket', 'auto']])
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 3d156eda..083be2b2 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -70,6 +70,15 @@ class DatabaseTest(LogCaptureTestCase):
 			return
 		self.assertEqual(self.dbFilename, self.db.filename)
 
+	def testPurgeAge(self):
+		if Fail2BanDb is None: # pragma: no cover
+			return
+		self.assertEqual(self.db.purgeage, 86400)
+		self.db.purgeage = '1y6mon15d5h30m'
+		self.assertEqual(self.db.purgeage, 48652200)
+		self.db.purgeage = '2y 12mon 30d 10h 60m'
+		self.assertEqual(self.db.purgeage, 48652200*2)
+
 	def testCreateInvalidPath(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 07e10c7d..a7d7adef 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -259,6 +259,7 @@ class Transmitter(TransmitterBase):
 	def testJailFindTime(self):
 		self.setGetTest("findtime", "120", 120, jail=self.jailName)
 		self.setGetTest("findtime", "60", 60, jail=self.jailName)
+		self.setGetTest("findtime", "30m", 30*60, jail=self.jailName)
 		self.setGetTest("findtime", "-60", -60, jail=self.jailName)
 		self.setGetTestNOK("findtime", "Dog", jail=self.jailName)
 
@@ -266,6 +267,7 @@ class Transmitter(TransmitterBase):
 		self.setGetTest("bantime", "600", 600, jail=self.jailName)
 		self.setGetTest("bantime", "50", 50, jail=self.jailName)
 		self.setGetTest("bantime", "-50", -50, jail=self.jailName)
+		self.setGetTest("bantime", "15d 5h 30m", 1315800, jail=self.jailName)
 		self.setGetTestNOK("bantime", "Cat", jail=self.jailName)
 
 	def testDatePattern(self):

From db5ee0842a56a5db0e933e0125b0d0ab4773c86a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Oct 2014 21:57:16 +0100
Subject: [PATCH 007/623] increase code coverage

---
 fail2ban/server/datedetector.py        |  2 +-
 fail2ban/server/jail.py                |  6 ++++--
 fail2ban/tests/datedetectortestcase.py | 13 +++++++++++--
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index dbfb9e5d..4bdcefd2 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -194,7 +194,7 @@ class DateDetector(object):
 						logSys.log(logLevel, "Got time %f for \"%r\" using template %s", 
 							date[0], date[1].group(), template.name)
 					return date
-				except ValueError:
+				except ValueError: # pragma: no cover
 					pass
 			return None
 		finally:
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index a866cb51..c5f2e9cd 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -76,7 +76,8 @@ class Jail:
 		self.__queue = Queue.Queue()
 		self.__filter = None
 		logSys.info("Creating new jail '%s'" % self.name)
-		self._setBackend(backend)
+		if backend is not None:
+			self._setBackend(backend)
 
 	def __repr__(self):
 		return "%s(%r)" % (self.__class__.__name__, self.name)
@@ -106,11 +107,12 @@ class Jail:
 					logSys.info("Initiated %r backend" % b)
 				self.__actions = Actions(self)
 				return					# we are done
-			except ImportError, e:
+			except ImportError, e: # pragma: no cover
 				# Log debug if auto, but error if specific
 				logSys.log(
 					logging.DEBUG if backend == "auto" else logging.ERROR,
 					"Backend %r failed to initialize due to %s" % (b, e))
+		# pragma: no cover
 		# log error since runtime error message isn't printed, INVALID COMMAND
 		logSys.error(
 			"Failed to initialize any backend for Jail %r" % self.name)
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 095b706a..9a7d3e78 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -29,20 +29,29 @@ import time
 import datetime
 
 from ..server.datedetector import DateDetector
+from ..server import datedetector
 from ..server.datetemplate import DateTemplate
-from .utils import setUpMyTime, tearDownMyTime
+from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase
+from ..helpers import getLogger
+
+logSys = getLogger("fail2ban")
 
 
-class DateDetectorTest(unittest.TestCase):
+class DateDetectorTest(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		LogCaptureTestCase.setUp(self)
+		self.__old_eff_level = datedetector.logLevel
+		datedetector.logLevel = logSys.getEffectiveLevel()
 		setUpMyTime()
 		self.__datedetector = DateDetector()
 		self.__datedetector.addDefaultTemplate()
 
 	def tearDown(self):
 		"""Call after every test case."""
+		LogCaptureTestCase.tearDown(self)
+		datedetector.logLevel = self.__old_eff_level
 		tearDownMyTime()
 	
 	def testGetEpochTime(self):

From d23d103f967440942ff598e7543153a54be3f738 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Tue, 2 Dec 2014 00:56:20 +0100
Subject: [PATCH 008/623] code review and few new test cases

---
 fail2ban/server/actions.py     |  1 +
 fail2ban/server/mytime.py      | 42 ++++++++++++++++++----------------
 fail2ban/tests/misctestcase.py | 18 +++++++++++++++
 fail2ban/tests/utils.py        |  3 ++-
 4 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index b4612f8c..c00dc7fc 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -167,6 +167,7 @@ class Actions(JailThread, Mapping):
 	# @param value the time
 	
 	def setBanTime(self, value):
+		value = MyTime.str2seconds(value)
 		self.__banManager.setBanTime(value)
 		logSys.info("Set banTime = %s" % value)
 	
diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py
index 47509f19..329e2465 100644
--- a/fail2ban/server/mytime.py
+++ b/fail2ban/server/mytime.py
@@ -98,32 +98,34 @@ class MyTime:
 		else:
 			return time.localtime(MyTime.myTime)
 
-	## 
-	# Wraps string expression like "1h 2m 3s" into number contains seconds (3723).
-	# The string expression will be evaluated as mathematical expression, spaces between each groups 
-	#   will be wrapped to "+" operand (only if any operand does not specified between).
-	# Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month
-	#   are "mo" or "mon" (like %b by date formating).
-	# Ex: 1hour+30min = 5400
-	#     0d 1h 30m   = 5400
-	#     1year-6mo   = 15778800
-	#     6 months    = 15778800
-	# warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also:
-	#       >>>> float(Test.str2seconds("1month")) / 60 / 60 / 24
-	#       30.4375
-	#       >>>> float(Test.str2seconds("1year")) / 60 / 60 / 24
-	#       365.25	
-	#
-	# @returns number (calculated seconds from expression "val")
-
 	@staticmethod
 	def str2seconds(val):
+		"""Wraps string expression like "1h 2m 3s" into number contains seconds (3723).
+		The string expression will be evaluated as mathematical expression, spaces between each groups 
+		  will be wrapped to "+" operand (only if any operand does not specified between).
+		Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month
+		  are "mo" or "mon".
+		Ex: 1hour+30min = 5400
+		    0d 1h 30m   = 5400
+		    1year-6mo   = 15778800
+		    6 months    = 15778800
+		warn: month is not 30 days, it is a year in seconds / 12, the leap years will be respected also:
+		      >>>> float(str2seconds("1month")) / 60 / 60 / 24
+		      30.4375
+		      >>>> float(str2seconds("1year")) / 60 / 60 / 24
+		      365.25	
+		
+		@returns number (calculated seconds from expression "val")
+		"""
+		if isinstance(val, (int, long, float, complex)):
+			return val
 		# replace together standing abbreviations, example '1d12h' -> '1d 12h':
 		val = re.sub(r"(?i)(?<=[a-z])(\d)", r" \1", val)
 		# replace abbreviation with expression:
 		for rexp, rpl in (
-			(r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), 
-			(r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60),
+			(r"days?|da|dd?", 24*60*60), (r"weeks?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), 
+			(r"years?|yea?|yy?", (365*3+366)*24*60*60/4), 
+			(r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|hou?|hh?", 60*60),
 		):
 			val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val)
 		val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val);
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index e28ce422..a2310ec5 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -35,6 +35,7 @@ from StringIO import StringIO
 from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger
 from ..helpers import splitcommaspace
 from ..server.datetemplate import DatePatternRegex
+from ..server.mytime import MyTime
 
 
 class HelpersTest(unittest.TestCase):
@@ -229,3 +230,20 @@ class CustomDateFormatsTest(unittest.TestCase):
 		self.assertEqual(
 			date,
 			datetime.datetime(2007, 1, 25, 16, 0))
+
+class MyTimeTest(unittest.TestCase):
+
+	def testStr2Seconds(self):
+		# several formats / write styles:
+		str2sec = MyTime.str2seconds
+		self.assertEqual(str2sec('1y6mo30w15d12h35m25s'), 66821725)
+		self.assertEqual(str2sec('2yy 3mo 4ww 10dd 5hh 30mm 20ss'), 74307620)
+		self.assertEqual(str2sec('2 years 3 months 4 weeks 10 days 5 hours 30 minutes 20 seconds'), 74307620)
+		self.assertEqual(str2sec('1 year + 1 month - 1 week + 1 day'), 33669000)
+		self.assertEqual(str2sec('2 * 0.5 yea + 1*1 mon - 3*1/3 wee + 2/2 day - (2*12 hou 3*20 min 80 sec) '), 33578920.0)
+		self.assertEqual(str2sec('2*.5y+1*1mo-3*1/3w+2/2d-(2*12h3*20m80s) '), 33578920.0)
+		self.assertEqual(str2sec('1ye -2mo -3we -4da -5ho -6mi -7se'), 24119633)
+		# month and year in days :
+		self.assertEqual(float(str2sec("1 month")) / 60 / 60 / 24, 30.4375)
+		self.assertEqual(float(str2sec("1 year")) / 60 / 60 / 24, 365.25)
+
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 8172e7ec..dff3faae 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -133,6 +133,7 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(misctestcase.SetupTest))
 	tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest))
 	tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest))
+	tests.addTest(unittest.makeSuite(misctestcase.MyTimeTest))
 	# Database
 	tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest))
 
@@ -300,4 +301,4 @@ else:
 			kernel32.CloseHandle(process)
 			return True
 		else:
-			return False
\ No newline at end of file
+			return False

From 3540619a736644180c603dd324c8242d0f4e9df4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 2 Dec 2014 11:57:43 +0100
Subject: [PATCH 009/623] code review, test case extended;

---
 fail2ban/server/filter.py        | 23 +++++++------
 fail2ban/tests/filtertestcase.py | 58 ++++++++++++++++++++++++++++++--
 2 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index ee0e2a1d..621f826b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -673,14 +673,19 @@ class FileFilter(Filter):
 			logSys.exception(e)
 			return False
 		except OSError, e: # pragma: no cover - Requires implemention error in FileContainer to generate
-			logSys.error("Internal errror in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
+			logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
 			logSys.exception(e)
 			return False
 
 		# prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm:
 		if log.getPos() == 0 and startTime is not None:
-			# startTime = MyTime.time() - self.getFindTime()
-			self.seekToTime(log, startTime)
+			try:
+				# startTime = MyTime.time() - self.getFindTime()
+				self.seekToTime(log, startTime)
+			except Exception, e: # pragma: no cover
+				logSys.error("Error during seek to start time in \"%s\"", filename)
+				logSys.exception(e)
+				return False
 
 		# yoh: has_content is just a bool, so do not expect it to
 		# change -- loop is exited upon break, and is not entered at
@@ -717,7 +722,7 @@ class FileFilter(Filter):
 		cntr = 0
 		unixTime = None
 		lasti = 0
-		movecntr = 3
+		movecntr = 1
 		while maxp > minp:
 			i = int(minp + (maxp - minp) / 2)
 			pos = container.seek(i)
@@ -726,7 +731,8 @@ class FileFilter(Filter):
 			lncntr = 5;
 			dateTimeMatch = None
 			llen = 0
-			i = pos
+			if lastpos == pos:
+				i = pos
 			while True:
 				line = container.readline()
 				if not line:
@@ -763,6 +769,7 @@ class FileFilter(Filter):
 			lastpos = container.seek(lastFew, False)
 		else:
 			lastpos = container.seek(lastpos, False)
+		container.setPos(lastpos)
 		if logSys.getEffectiveLevel() <= logging.DEBUG:
 			logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, 
 				(datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr)
@@ -952,10 +959,6 @@ class DNSUtils:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s"
 						% (dns, e))
 			return list()
-		except socket.error, e:
-			logSys.warning("Socket error raised trying to resolve hostname %s: %s"
-						% (dns, e))
-			return list()
 
 	@staticmethod
 	def ipToName(ip):
@@ -984,7 +987,7 @@ class DNSUtils:
 		try:
 			socket.inet_aton(s[0])
 			return True
-		except socket.error:
+		except socket.error: # pragma: no cover
 			return False
 
 	@staticmethod
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index c7dba5ce..7a35bdd9 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -27,7 +27,7 @@ import unittest
 import getpass
 import os
 import sys
-import time
+import time, datetime
 import tempfile
 import uuid
 
@@ -38,7 +38,7 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, DNSUtils
+from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils
 from ..server.failmanager import FailManagerEmpty
 from ..server.mytime import MyTime
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
@@ -346,6 +346,60 @@ class LogFileFilterPoll(unittest.TestCase):
 		self.assertTrue(self.filter.isModified(LogFileFilterPoll.FILENAME))
 		self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME))
 
+	def testSeekToTime(self):
+		fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log')
+		tm = lambda time: datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S")
+		time = 1417512352
+		f = open(fname, 'w')
+		fc = FileContainer(fname, self.filter.getLogEncoding())
+		fc.open()
+		fc.setPos(0); self.filter.seekToTime(fc, time)
+		try:
+			f.flush()
+			# empty :
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 0)
+			# one entry with exact time:
+			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			# one entry with smaller time:
+			f.seek(0)
+			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time - 10))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 0)
+			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time - 9))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 0)
+			# add exact time between:
+			f.write("%s [sshd] error: PAM: failure\n" % tm(time - 1))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 110)
+			# stil one exact line:
+			f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time))
+			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 110)
+			# add something hereafter:
+			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 2))
+			f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time + 3))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 110)
+			# add something hereafter:
+			f.write("%s [sshd] error: PAM: failure\n" % tm(time + 9))
+			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 9))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 110)
+
+		finally:
+			fc.close()
+			_killfile(f, fname)
 
 class LogFileMonitor(LogCaptureTestCase):
 	"""Few more tests for FilterPoll API

From 59bf5013c0f7ddcfa88ee638031ec6f79c97393e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Jul 2015 14:58:00 +0200
Subject: [PATCH 010/623] - performance of fail2ban optimized    -- cache
 dnsToIp, ipToName to prevent long wait during retrieving of ip/name for wrong
 dns or lazy dns-system;    -- instead of simple "sleep" used conditional wait
 "wait_for", that internal increases sleep interval up to sleeptime;    --
 ticket / banmanager / failmanager modules are performance optimized;    --
 api of filter (log files), jail, etc. rewritten and extended for performance
 purposes; - performance of test cases optimized:    -- added option "--fast"
 to decrease wait intervals, avoid passive waiting, and skip few very slow
 test cases; - code review after partially cherry pick of branch
 'ban-time-incr' (see gh-716)    -- ticket module prepared to easy merge with
 newest version of 'ban-time-incr', now additionally holds banTime, banCount
 and json-data;    -- executeCmd partially moved from action to new module
 utils, etc.    -- python 2.6 compatibility; -
 testExecuteTimeoutWithNastyChildren: test case repaired - wait for pid file
 inside bash, kill tree in any case (gh-1155); - testSocket: test case
 repaired - wait for server thread starts a socket (listener)

---
 MANIFEST                                      |   2 +-
 bin/fail2ban-testcases                        |   5 +-
 doc/fail2ban.server.rst                       |   2 +-
 ...faildata.rst => fail2ban.server.utils.rst} |   4 +-
 fail2ban/server/action.py                     |  73 +-----
 fail2ban/server/actions.py                    |  12 +-
 fail2ban/server/banmanager.py                 |  37 ++-
 fail2ban/server/database.py                   |  13 +-
 fail2ban/server/faildata.py                   |  71 -----
 fail2ban/server/failmanager.py                |  38 +--
 fail2ban/server/filter.py                     |  70 +++--
 fail2ban/server/filtergamin.py                |  26 +-
 fail2ban/server/filterpoll.py                 |  68 ++---
 fail2ban/server/jailthread.py                 |   7 +-
 fail2ban/server/server.py                     |  12 +-
 fail2ban/server/ticket.py                     | 145 +++++++++--
 fail2ban/server/transmitter.py                |   2 +-
 fail2ban/server/utils.py                      | 242 ++++++++++++++++++
 fail2ban/tests/action_d/test_badips.py        |   2 +
 fail2ban/tests/action_d/test_smtp.py          |   2 +
 fail2ban/tests/actionstestcase.py             |  10 +-
 fail2ban/tests/actiontestcase.py              |  59 +++--
 fail2ban/tests/banmanagertestcase.py          |   1 +
 fail2ban/tests/databasetestcase.py            |  13 +-
 fail2ban/tests/dummyjail.py                   |  23 +-
 fail2ban/tests/filtertestcase.py              |  70 ++---
 fail2ban/tests/servertestcase.py              |  47 ++--
 fail2ban/tests/sockettestcase.py              |  17 +-
 fail2ban/tests/utils.py                       |  85 +++---
 29 files changed, 768 insertions(+), 390 deletions(-)
 rename doc/{fail2ban.server.faildata.rst => fail2ban.server.utils.rst} (55%)
 delete mode 100644 fail2ban/server/faildata.py
 create mode 100644 fail2ban/server/utils.py

diff --git a/MANIFEST b/MANIFEST
index d6bdba10..7306cc41 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -180,7 +180,6 @@ fail2ban/server/banmanager.py
 fail2ban/server/database.py
 fail2ban/server/datedetector.py
 fail2ban/server/datetemplate.py
-fail2ban/server/faildata.py
 fail2ban/server/failmanager.py
 fail2ban/server/failregex.py
 fail2ban/server/filter.py
@@ -197,6 +196,7 @@ fail2ban/server/server.py
 fail2ban/server/strptime.py
 fail2ban/server/ticket.py
 fail2ban/server/transmitter.py
+fail2ban/server/utils.py
 fail2ban/tests/__init__.py
 fail2ban/tests/action_d/__init__.py
 fail2ban/tests/action_d/test_badips.py
diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index dd6547a5..28af8265 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -58,6 +58,9 @@ def get_opt_parser():
 		Option('-n', "--no-network", action="store_true",
 			   dest="no_network",
 			   help="Do not run tests that require the network"),
+		Option('-f', "--fast", action="store_true",
+			   dest="fast",
+			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
 		Option("-t", "--log-traceback", action='store_true',
 			   help="Enrich log-messages with compressed tracebacks"),
 		Option("--full-traceback", action='store_true',
@@ -120,7 +123,7 @@ if not opts.log_level or opts.log_level != 'critical': # pragma: no cover
 	print("Fail2ban %s test suite. Python %s. Please wait..." \
 	    % (version, str(sys.version).replace('\n', '')))
 
-tests = gatherTests(regexps, opts.no_network)
+tests = gatherTests(regexps, opts)
 #
 # Run the tests
 #
diff --git a/doc/fail2ban.server.rst b/doc/fail2ban.server.rst
index fdc3bd20..dc5d1dee 100644
--- a/doc/fail2ban.server.rst
+++ b/doc/fail2ban.server.rst
@@ -10,7 +10,6 @@ fail2ban.server package
    fail2ban.server.database
    fail2ban.server.datedetector
    fail2ban.server.datetemplate
-   fail2ban.server.faildata
    fail2ban.server.failmanager
    fail2ban.server.failregex
    fail2ban.server.filter
@@ -26,3 +25,4 @@ fail2ban.server package
    fail2ban.server.strptime
    fail2ban.server.ticket
    fail2ban.server.transmitter
+   fail2ban.server.utils
diff --git a/doc/fail2ban.server.faildata.rst b/doc/fail2ban.server.utils.rst
similarity index 55%
rename from doc/fail2ban.server.faildata.rst
rename to doc/fail2ban.server.utils.rst
index c11d8208..3061e632 100644
--- a/doc/fail2ban.server.faildata.rst
+++ b/doc/fail2ban.server.utils.rst
@@ -1,7 +1,7 @@
-fail2ban.server.faildata module
+fail2ban.server.utils module
 ===============================
 
-.. automodule:: fail2ban.server.faildata
+.. automodule:: fail2ban.server.utils
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 85bdd166..446ae853 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -32,6 +32,7 @@ import time
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -40,21 +41,6 @@ logSys = getLogger(__name__)
 # Create a lock for running system commands
 _cmd_lock = threading.Lock()
 
-# Some hints on common abnormal exit codes
-_RETCODE_HINTS = {
-	127: '"Command not found".  Make sure that all commands in %(realCmd)r '
-			'are in the PATH of fail2ban-server process '
-			'(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). '
-			'You may want to start '
-			'"fail2ban-server -f" separately, initiate it with '
-			'"fail2ban-client reload" in another shell session and observe if '
-			'additional informative error messages appear in the terminals.'
-	}
-
-# Dictionary to lookup signal name from number
-signame = dict((num, name)
-	for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))
-
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -561,61 +547,6 @@ class CommandAction(ActionBase):
 
 		_cmd_lock.acquire()
 		try:
-			retcode = None  # to guarantee being defined upon early except
-			stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_")
-			stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_")
-
-			popen = subprocess.Popen(
-				realCmd, stdout=stdout, stderr=stderr, shell=True,
-				preexec_fn=os.setsid  # so that killpg does not kill our process
-			)
-			stime = time.time()
-			retcode = popen.poll()
-			while time.time() - stime <= timeout and retcode is None:
-				time.sleep(0.1)
-				retcode = popen.poll()
-			if retcode is None:
-				logSys.error("%s -- timed out after %i seconds." %
-				    (realCmd, timeout))
-				pgid = os.getpgid(popen.pid)
-				os.killpg(pgid, signal.SIGTERM)  # Terminate the process
-				time.sleep(0.1)
-				retcode = popen.poll()
-				if retcode is None:  # Still going...
-					os.killpg(pgid, signal.SIGKILL)  # Kill the process
-					time.sleep(0.1)
-					retcode = popen.poll()
-		except OSError as e:
-			logSys.error("%s -- failed with %s" % (realCmd, e))
+			return Utils.executeCmd(realCmd, timeout, shell=True, output=False)
 		finally:
 			_cmd_lock.release()
-
-		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
-		if std_level >= logSys.getEffectiveLevel():
-			stdout.seek(0); msg = stdout.read()
-			if msg != '':
-				logSys.log(std_level, "%s -- stdout: %r", realCmd, msg)
-			stderr.seek(0); msg = stderr.read()
-			if msg != '':
-				logSys.log(std_level, "%s -- stderr: %r", realCmd, msg)
-		stdout.close()
-		stderr.close()
-
-		if retcode == 0:
-			logSys.debug("%s -- returned successfully" % realCmd)
-			return True
-		elif retcode is None:
-			logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid))
-		elif retcode < 0 or retcode > 128:
-			# dash would return negative while bash 128 + n
-			sigcode = -retcode if retcode < 0 else retcode - 128
-			logSys.error("%s -- killed with %s (return code: %s)" %
-				(realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode))
-		else:
-			msg = _RETCODE_HINTS.get(retcode, None)
-			logSys.error("%s -- returned %i" % (realCmd, retcode))
-			if msg:
-				logSys.info("HINT on %i: %s"
-							% (retcode, msg % locals()))
-		return False
-
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index c00dc7fc..5469722f 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -42,6 +42,7 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -225,14 +226,11 @@ class Actions(JailThread, Mapping):
 					self._jail.name, name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
-			if not self.idle:
-				#logSys.debug(self._jail.name + ": action")
-				ret = self.__checkBan()
-				if not ret:
-					self.__checkUnBan()
-					time.sleep(self.sleeptime)
-			else:
+			if self.idle:
 				time.sleep(self.sleeptime)
+				continue
+			if not Utils.wait_for(self.__checkBan, self.sleeptime):
+				self.__checkUnBan()
 		self.__flushBan()
 
 		actions = self._actions.items()
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 662666b0..a958a5c8 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -247,12 +247,10 @@ class BanManager:
 	
 	@staticmethod
 	def createBanTicket(ticket):
-		ip = ticket.getIP()
-		#lastTime = ticket.getTime()
-		lastTime = MyTime.time()
-		banTicket = BanTicket(ip, lastTime, ticket.getMatches())
-		banTicket.setAttempt(ticket.getAttempt())
-		return banTicket
+		# we should always use correct time to calculate correct end time (ban time is variable now, 
+		# + possible double banning by restore from database and from log file)
+		# so use as lastTime always time from ticket.
+		return BanTicket(ticket=ticket)
 	
 	##
 	# Add a ban ticket.
@@ -264,11 +262,25 @@ class BanManager:
 	def addBanTicket(self, ticket):
 		try:
 			self.__lock.acquire()
-			if not self._inBanList(ticket):
-				self.__banList.append(ticket)
-				self.__banTotal += 1
-				return True
-			return False
+			# check already banned
+			for i in self.__banList:
+				if ticket.getIP() == i.getIP():
+					# if already permanent
+					btorg, torg = i.getBanTime(self.__banTime), i.getTime()
+					if btorg == -1:
+						return False
+					# if given time is less than already banned time
+					btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
+					if btnew != -1 and tnew + btnew <= torg + btorg:
+						return False
+					# we have longest ban - set new (increment) ban time
+					i.setTime(tnew)
+					i.setBanTime(btnew)
+					return False
+			# not yet banned - add new
+			self.__banList.append(ticket)
+			self.__banTotal += 1
+			return True
 		finally:
 			self.__lock.release()
 
@@ -313,8 +325,7 @@ class BanManager:
 				return list()
 
 			# Gets the list of ticket to remove.
-			unBanList = [ticket for ticket in self.__banList
-						 if ticket.getTime() < time - self.__banTime]
+			unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)]
 			
 			# Removes tickets.
 			self.__banList = [ticket for ticket in self.__banList
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 3cfaa5a6..c6e8c95c 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -418,8 +418,7 @@ class Fail2BanDb(object):
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
 			(jail.name, ticket.getIP(), int(round(ticket.getTime())),
-				{"matches": ticket.getMatches(),
-				 "failures": ticket.getAttempt()}))
+				ticket.getData()))
 
 	@commitandrollback
 	def delBan(self, cur, jail, ip):
@@ -477,8 +476,8 @@ class Fail2BanDb(object):
 		tickets = []
 		for ip, timeofban, data in self._getBans(**kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
-			tickets.append(FailTicket(ip, timeofban, data.get('matches')))
-			tickets[-1].setAttempt(data.get('failures', 1))
+			tickets.append(FailTicket(ip, timeofban))
+			tickets[-1].setData(data)
 		return tickets
 
 	def getBansMerged(self, ip=None, jail=None, bantime=None):
@@ -520,6 +519,7 @@ class Fail2BanDb(object):
 				prev_banip = results[0][0]
 				matches = []
 				failures = 0
+				tickdata = {}
 				for banip, timeofban, data in results:
 					#TODO: Implement data parts once arbitrary match keys completed
 					if banip != prev_banip:
@@ -530,11 +530,14 @@ class Fail2BanDb(object):
 						prev_banip = banip
 						matches = []
 						failures = 0
-					matches.extend(data.get('matches', []))
+						tickdata = {}
+					matches.extend(data.get('matches', ()))
 					failures += data.get('failures', 1)
+					tickdata.update(data.get('data', {}))
 					prev_timeofban = timeofban
 				ticket = FailTicket(banip, prev_timeofban, matches)
 				ticket.setAttempt(failures)
+				ticket.setData(**tickdata)
 				tickets.append(ticket)
 
 			if cacheKey:
diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py
deleted file mode 100644
index 2dd8d4d8..00000000
--- a/fail2ban/server/faildata.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
-# vi: set ft=python sts=4 ts=4 sw=4 noet :
-
-# This file is part of Fail2Ban.
-#
-# Fail2Ban is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# Fail2Ban is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Fail2Ban; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-# Author: Cyril Jaquier
-# 
-
-__author__ = "Cyril Jaquier"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
-__license__ = "GPL"
-
-from ..helpers import getLogger
-
-# Gets the instance of the logger.
-logSys = getLogger(__name__)
-
-
-class FailData:
-	
-	def __init__(self):
-		self.__retry = 0
-		self.__lastTime = 0
-		self.__lastReset = 0
-		self.__matches = []
-
-	def setRetry(self, value):
-		self.__retry = value
-		# keep only the last matches or reset entirely
-		# Explicit if/else for compatibility with Python 2.4
-		if value:
-			self.__matches = self.__matches[-min(len(self.__matches, value)):]
-		else:
-			self.__matches = []
-
-	def getRetry(self):
-		return self.__retry
-
-	def getMatches(self):
-		return self.__matches
-
-	def inc(self, matches=None):
-		self.__retry += 1
-		self.__matches += matches or []
-
-	def setLastTime(self, value):
-		if value > self.__lastTime:
-			self.__lastTime = value
-	
-	def getLastTime(self):
-		return self.__lastTime
-
-	def getLastReset(self):
-		return self.__lastReset
-
-	def setLastReset(self, value):
-		self.__lastReset = value
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 37a5fe55..4d006098 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -27,7 +27,6 @@ __license__ = "GPL"
 from threading import Lock
 import logging
 
-from .faildata import FailData
 from .ticket import FailTicket
 from ..helpers import getLogger
 
@@ -86,26 +85,35 @@ class FailManager:
 		finally:
 			self.__lock.release()
 
-	def addFailure(self, ticket):
+	def addFailure(self, ticket, count=1):
+		attempts = 1
 		try:
 			self.__lock.acquire()
 			ip = ticket.getIP()
-			unixTime = ticket.getTime()
-			matches = ticket.getMatches()
 			if ip in self.__failList:
 				fData = self.__failList[ip]
+				# if the same object:
+				if fData is ticket:
+					matches = None
+				else:
+					matches = ticket.getMatches()
+				unixTime = ticket.getTime()
 				if fData.getLastReset() < unixTime - self.__maxTime:
 					fData.setLastReset(unixTime)
 					fData.setRetry(0)
-				fData.inc(matches)
+				fData.inc(matches, 1, count)
 				fData.setLastTime(unixTime)
 			else:
-				fData = FailData()
-				fData.inc(matches)
-				fData.setLastReset(unixTime)
-				fData.setLastTime(unixTime)
+				# if already FailTicket - add it direct, otherwise create (using copy all ticket data):
+				if isinstance(ticket, FailTicket):
+					fData = ticket;
+				else:
+					fData = FailTicket(ticket=ticket)
+				if count > ticket.getAttempt():
+					fData.setRetry(count)
 				self.__failList[ip] = fData
 
+			attempts = fData.getRetry()
 			self.__failTotal += 1
 
 			if logSys.getEffectiveLevel() <= logging.DEBUG:
@@ -118,6 +126,7 @@ class FailManager:
 							 % (self.__failTotal, len(self.__failList), failures_summary))
 		finally:
 			self.__lock.release()
+		return attempts
 	
 	def size(self):
 		try:
@@ -140,17 +149,14 @@ class FailManager:
 		if ip in self.__failList:
 			del self.__failList[ip]
 	
-	def toBan(self):
+	def toBan(self, ip=None):
 		try:
 			self.__lock.acquire()
-			for ip in self.__failList:
+			for ip in ([ip] if ip != None and ip in self.__failList else self.__failList):
 				data = self.__failList[ip]
 				if data.getRetry() >= self.__maxRetry:
-					self.__delFailure(ip)
-					# Create a FailTicket from BanData
-					failTicket = FailTicket(ip, data.getLastTime(), data.getMatches())
-					failTicket.setAttempt(data.getRetry())
-					return failTicket
+					del self.__failList[ip]
+					return data
 			raise FailManagerEmpty
 		finally:
 			self.__lock.release()
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 621f826b..0a2a9949 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
 __license__ = "GPL"
 
 import codecs
+import datetime
 import fcntl
 import locale
 import logging
@@ -316,13 +317,12 @@ class Filter(JailThread):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
 		unixTime = MyTime.time()
-		for i in xrange(self.failManager.getMaxRetry()):
-			self.failManager.addFailure(FailTicket(ip, unixTime))
+		self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry())
 
 		# Perform the banning of the IP now.
 		try: # pragma: no branch - exception is the only way out
 			while True:
-				ticket = self.failManager.toBan()
+				ticket = self.failManager.toBan(ip)
 				self.jail.putFailTicket(ticket)
 		except FailManagerEmpty:
 			self.failManager.cleanup(MyTime.time())
@@ -427,17 +427,19 @@ class Filter(JailThread):
 			ip = element[1]
 			unixTime = element[2]
 			lines = element[3]
-			logSys.debug("Processing line with time:%s and ip:%s"
-						 % (unixTime, ip))
+			logSys.debug("Processing line with time:%s and ip:%s", 
+					unixTime, ip)
 			if unixTime < MyTime.time() - self.getFindTime():
-				logSys.debug("Ignore line since time %s < %s - %s"
-							 % (unixTime, MyTime.time(), self.getFindTime()))
+				logSys.debug("Ignore line since time %s < %s - %s", 
+					unixTime, MyTime.time(), self.getFindTime())
 				break
 			if self.inIgnoreIPList(ip, log_ignore=True):
 				continue
-			logSys.info("[%s] Found %s" % (self.jail.name, ip))
-			## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),)
-			self.failManager.addFailure(FailTicket(ip, unixTime, lines))
+			logSys.info(
+				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
+			)
+			tick = FailTicket(ip, unixTime, lines)
+			self.failManager.addFailure(tick)
 
 	##
 	# Returns true if the line should be ignored.
@@ -606,6 +608,14 @@ class FileFilter(Filter):
 		# to be overridden by backends
 		pass
 
+	##
+	# Get the log file names
+	#
+	# @return log paths
+
+	def getLogPaths(self):
+		return self.__logs.keys()
+
 	##
 	# Get the log containers
 	#
@@ -614,6 +624,14 @@ class FileFilter(Filter):
 	def getLogs(self):
 		return self.__logs.values()
 
+	##
+	# Get the count of log containers
+	#
+	# @return count of log containers
+
+	def getLogCount(self):
+		return len(self.__logs)
+
 	##
 	# Check whether path is already monitored.
 	#
@@ -941,32 +959,50 @@ class JournalFilter(Filter): # pragma: systemd no cover
 
 import socket
 import struct
+from .utils import Utils
 
 
 class DNSUtils:
 
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=60*60)
+	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=60*60)
+
 	@staticmethod
 	def dnsToIp(dns):
 		""" Convert a DNS into an IP address using the Python socket module.
 			Thanks to Kevin Drapel.
 		"""
+		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
+		v = DNSUtils.CACHE_dnsToIp.get(dns)
+		if v is not None: 
+			return v
 		# retrieve ip (todo: use AF_INET6 for IPv6)
 		try:
-			return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
+			v = set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
 		except socket.error, e:
-			logSys.warning("Unable to find a corresponding IP address for %s: %s"
-						% (dns, e))
-			return list()
+			# todo: make configurable the expired time of cache entry:
+			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
+			v = list()
+		DNSUtils.CACHE_dnsToIp.set(dns, v)
+		return v
 
 	@staticmethod
 	def ipToName(ip):
+		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
+		v = DNSUtils.CACHE_ipToName.get(ip)
+		if v is not None: 
+			return v
+		# retrieve name
 		try:
-			return socket.gethostbyaddr(ip)[0]
+			v = socket.gethostbyaddr(ip)[0]
 		except socket.error, e:
-			logSys.debug("Unable to find a name for the IP %s: %s" % (ip, e))
-			return None
+			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
+			v = None
+		DNSUtils.CACHE_ipToName.set(ip, v)
+		return v
 
 	@staticmethod
 	def searchIP(text):
diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index e731a8e9..b094cfeb 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -31,6 +31,7 @@ import gamin
 from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -102,6 +103,15 @@ class FilterGamin(FileFilter):
 	def _delLogPath(self, path):
 		self.monitor.stop_watch(path)
 
+	def _handleEvents(self):
+		ret = False
+		mon = self.monitor
+		while mon and mon.event_pending():
+			mon.handle_events()
+			mon = self.monitor
+			ret = True
+		return ret
+
 	##
 	# Main loop.
 	#
@@ -112,12 +122,10 @@ class FilterGamin(FileFilter):
 	def run(self):
 		# Gamin needs a loop to collect and dispatch events
 		while self.active:
-			if not self.idle:
-				# We cannot block here because we want to be able to
-				# exit.
-				if self.monitor.event_pending():
-					self.monitor.handle_events()
-			time.sleep(self.sleeptime)
+			if self.idle:
+				time.sleep(self.sleeptime)
+				continue
+			Utils.wait_for(self._handleEvents, self.sleeptime)
 		logSys.debug(self.jail.name + ": filter terminated")
 		return True
 
@@ -129,6 +137,6 @@ class FilterGamin(FileFilter):
 	# Desallocates the resources used by Gamin.
 
 	def __cleanup(self):
-		for log in self.getLogs():
-			self.monitor.stop_watch(log.getFileName())
-		del self.monitor
+		for filename in self.getLogPaths():
+			self.monitor.stop_watch(filename)
+		self.monitor = None
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 0e99455e..1afda7f2 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -31,6 +31,7 @@ from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime
 from ..helpers import getLogger
+from ..server.utils import Utils
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -78,6 +79,15 @@ class FilterPoll(FileFilter):
 		del self.__prevStats[path]
 		del self.__file404Cnt[path]
 
+	##
+	# Get a modified log path at once
+	#
+	def getModified(self, modlst):
+		for filename in self.getLogPaths():
+			if self.isModified(filename):
+				modlst.append(filename)
+		return modlst
+
 	##
 	# Main loop.
 	#
@@ -89,31 +99,32 @@ class FilterPoll(FileFilter):
 		while self.active:
 			if logSys.getEffectiveLevel() <= 6:
 				logSys.log(6, "Woke up idle=%s with %d files monitored",
-						   self.idle, len(self.getLogs()))
-			if not self.idle:
-				# Get file modification
-				for container in self.getLogs():
-					filename = container.getFileName()
-					if self.isModified(filename):
-						# set start time as now - find time for first usage only (prevent performance bug with polling of big files)
-						self.getFailures(filename, 
-							(MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None
-						)
-						self.__initial[filename] = True
-						self.__modified = True
+						   self.idle, self.getLogCount())
+			if self.idle:
+				if not Utils.wait_for(lambda: not self.idle, 
+					self.sleeptime * 100, self.sleeptime
+				):
+					continue
+			# Get file modification
+			modlst = []
+			Utils.wait_for(lambda: self.getModified(modlst), self.sleeptime)
+			for filename in modlst:
+				# set start time as now - find time for first usage only (prevent performance bug with polling of big files)
+				self.getFailures(filename, 
+					(MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None
+				)
+				self.__initial[filename] = True
+				self.__modified = True
 
-				if self.__modified:
-					try:
-						while True:
-							ticket = self.failManager.toBan()
-							self.jail.putFailTicket(ticket)
-					except FailManagerEmpty:
-						self.failManager.cleanup(MyTime.time())
-					self.dateDetector.sortTemplate()
-					self.__modified = False
-				time.sleep(self.sleeptime)
-			else:
-				time.sleep(self.sleeptime)
+			if self.__modified:
+				try:
+					while True:
+						ticket = self.failManager.toBan()
+						self.jail.putFailTicket(ticket)
+				except FailManagerEmpty:
+					self.failManager.cleanup(MyTime.time())
+				self.dateDetector.sortTemplate()
+				self.__modified = False
 		logSys.debug(
 			(self.jail is not None and self.jail.name or "jailless") +
 					 " filter terminated")
@@ -129,7 +140,7 @@ class FilterPoll(FileFilter):
 		try:
 			logStats = os.stat(filename)
 			stats = logStats.st_mtime, logStats.st_ino, logStats.st_size
-			pstats = self.__prevStats[filename]
+			pstats = self.__prevStats.get(filename, ())
 			self.__file404Cnt[filename] = 0
 			if logSys.getEffectiveLevel() <= 7:
 				# we do not want to waste time on strftime etc if not necessary
@@ -139,10 +150,9 @@ class FilterPoll(FileFilter):
 				# os.system("stat %s | grep Modify" % filename)
 			if pstats == stats:
 				return False
-			else:
-				logSys.debug("%s has been modified", filename)
-				self.__prevStats[filename] = stats
-				return True
+			logSys.debug("%s has been modified", filename)
+			self.__prevStats[filename] = stats
+			return True
 		except OSError, e:
 			logSys.error("Unable to get stat on %s because of: %s"
 						 % (filename, e))
diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py
index 3897801a..eb43e453 100644
--- a/fail2ban/server/jailthread.py
+++ b/fail2ban/server/jailthread.py
@@ -28,6 +28,7 @@ import sys
 from threading import Thread
 from abc import abstractmethod
 
+from .utils import Utils
 from ..helpers import excepthook
 
 
@@ -48,14 +49,14 @@ class JailThread(Thread):
 		The time the thread sleeps for in the loop.
 	"""
 
-	def __init__(self):
-		super(JailThread, self).__init__()
+	def __init__(self, name=None):
+		super(JailThread, self).__init__(name=name)
 		## Control the state of the thread.
 		self.active = False
 		## Control the idle state of the thread.
 		self.idle = False
 		## The time the thread sleeps in the loop.
-		self.sleeptime = 1
+		self.sleeptime = Utils.DEFAULT_SLEEP_TIME
 
 		# excepthook workaround for threads, derived from:
 		# http://bugs.python.org/issue1230540#msg91244
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 9836b61d..1b335d02 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -211,8 +211,7 @@ class Server:
 	def getLogPath(self, name):
 		filter_ = self.__jails[name].filter
 		if isinstance(filter_, FileFilter):
-			return [m.getFileName()
-					for m in filter_.getLogs()]
+			return filter_.getLogPaths()
 		else: # pragma: systemd no cover
 			logSys.info("Jail %s is not a FileFilter instance" % name)
 			return []
@@ -324,6 +323,15 @@ class Server:
 	def getBanTime(self, name):
 		return self.__jails[name].actions.getBanTime()
 	
+	def is_alive(self, jailnum=None):
+		if jailnum is not None and len(self.__jails) != jailnum:
+			return 0
+		for j in self.__jails:
+			j = self.__jails[j]
+			if not j.is_alive():
+				return 0
+		return 1
+
 	# Status
 	def status(self):
 		try:
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 70be06fe..49ebf9ea 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -24,7 +24,10 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
+import sys
+
 from ..helpers import getLogger
+from .mytime import MyTime
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -32,7 +35,7 @@ logSys = getLogger(__name__)
 
 class Ticket:
 	
-	def __init__(self, ip, time, matches=None):
+	def __init__(self, ip=None, time=None, matches=None, ticket=None):
 		"""Ticket constructor
 
 		@param ip the IP address
@@ -41,14 +44,21 @@ class Ticket:
 		"""
 
 		self.setIP(ip)
-		self.__time = time
-		self.__attempt = 0
-		self.__file = None
-		self.__matches = matches or []
+		self._flags = 0;
+		self._banCount = 0;
+		self._banTime = None;
+		self._time = time if time is not None else MyTime.time()
+		self._data = {'matches': [], 'failures': 0}
+		if ticket:
+			# ticket available - copy whole information from ticket:
+			self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__)
+		else:
+			self._data['matches'] = matches or []
 
 	def __str__(self):
 		return "%s: ip=%s time=%s #attempts=%d matches=%r" % \
-			   (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches)
+			   (self.__class__.__name__.split('.')[-1], self.__ip, self._time,
+			   	self._data['failures'], self._data.get('matches', []))
 
 	def __repr__(self):
 		return str(self)
@@ -56,9 +66,8 @@ class Ticket:
 	def __eq__(self, other):
 		try:
 			return self.__ip == other.__ip and \
-				round(self.__time, 2) == round(other.__time, 2) and \
-				self.__attempt == other.__attempt and \
-				self.__matches == other.__matches
+				round(self._time, 2) == round(other._time, 2) and \
+				self._data == other._data
 		except AttributeError:
 			return False
 
@@ -72,24 +81,128 @@ class Ticket:
 		return self.__ip
 	
 	def setTime(self, value):
-		self.__time = value
+		self._time = value
 	
 	def getTime(self):
-		return self.__time
-	
+		return self._time
+
+	def setBanTime(self, value):
+		self._banTime = value;
+
+	def getBanTime(self, defaultBT = None):
+		return (self._banTime if not self._banTime is None else defaultBT);
+
+	def setBanCount(self, value):
+		self._banCount = value;
+
+	def incrBanCount(self, value = 1):
+		self._banCount += value;
+
+	def getBanCount(self):
+		return self._banCount;
+
+	def isTimedOut(self, time, defaultBT = None):
+		bantime = (self._banTime if not self._banTime is None else defaultBT);
+		# permanent
+		if bantime == -1:
+			return False
+		# timed out
+		return (time > self._time + bantime)
+
 	def setAttempt(self, value):
-		self.__attempt = value
+		self._data['failures'] = value
 	
 	def getAttempt(self):
-		return self.__attempt
+		return self._data['failures']
+
+	def setMatches(self, matches):
+		self._data['matches'] = matches or []
 
 	def getMatches(self):
-		return self.__matches
+		return self._data.get('matches', [])
+
+	def setData(self, *args, **argv):
+		# if overwrite - set data and filter None values:
+		if len(args) == 1:
+			# todo: if support >= 2.7 only:
+			# self._data = {k:v for k,v in args[0].iteritems() if v is not None}
+			self._data = dict([(k,v) for k,v in args[0].iteritems() if v is not None])
+		# add k,v list or dict (merge):
+		elif len(args) == 2:
+			self._data.update((args,))
+		elif len(args) > 2:
+			self._data.update((k,v) for k,v in zip(*[iter(args)]*2))
+		if len(argv):
+			self._data.update(argv)
+		# filter (delete) None values:
+		# todo: if support >= 2.7 only:
+		# self._data = {k:v for k,v in self._data.iteritems() if v is not None}
+		self._data = dict([(k,v) for k,v in self._data.iteritems() if v is not None])
+	
+	def getData(self, key=None, default=None):
+		# return whole data dict:
+		if key is None:
+			return self._data
+		# return default if not exists:
+		if not self._data:
+			return default
+		# return filtered by lambda/function:
+		if callable(key):
+			# todo: if support >= 2.7 only:
+			# return {k:v for k,v in self._data.iteritems() if key(k)}
+			return dict([(k,v) for k,v in self._data.iteritems() if key(k)])
+		# return filtered by keys:
+		if hasattr(key, '__iter__'):
+			# todo: if support >= 2.7 only:
+			# return {k:v for k,v in self._data.iteritems() if k in key}
+			return dict([(k,v) for k,v in self._data.iteritems() if k in key])
+		# return single value of data:
+		return self._data.get(key, default)
 
 
 class FailTicket(Ticket):
-	pass
 
+	def __init__(self, ip=None, time=None, matches=None, ticket=None):
+		# this class variables:
+		self.__retry = 0
+		self.__lastReset = None
+		# create/copy using default ticket constructor:
+		Ticket.__init__(self, ip, time, matches, ticket)
+		# init:
+		if ticket is None:
+			self.__lastReset = time if time is not None else self.getTime()
+		if not self.__retry:
+			self.__retry = self._data['failures'];
+
+	def setRetry(self, value):
+		self.__retry = value
+		if not self._data['failures']:
+			self._data['failures'] = 1
+		if not value:
+			self._data['failures'] = 0
+			self._data['matches'] = []
+
+	def getRetry(self):
+		return max(self.__retry, self._data['failures'])
+
+	def inc(self, matches=None, attempt=1, count=1):
+		self.__retry += count
+		self._data['failures'] += attempt
+		if matches:
+			self._data['matches'] += matches
+
+	def setLastTime(self, value):
+		if value > self._time:
+			self._time = value
+	
+	def getLastTime(self):
+		return self._time
+
+	def getLastReset(self):
+		return self.__lastReset
+
+	def setLastReset(self, value):
+		self.__lastReset = value
 
 ##
 # Ban Ticket.
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index eb5b7713..4c4c32f7 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -95,7 +95,7 @@ class Transmitter:
 			return None
 		elif command[0] == "sleep":
 			value = command[1]
-			time.sleep(int(value))
+			time.sleep(float(value))
 			return None
 		elif command[0] == "flushlogs":
 			return self.__server.flushLogs()
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
new file mode 100644
index 00000000..7e69ddca
--- /dev/null
+++ b/fail2ban/server/utils.py
@@ -0,0 +1,242 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors"
+__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester"
+__license__ = "GPL"
+
+import logging, os, fcntl, subprocess, time, signal
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger(__name__)
+
+# Some hints on common abnormal exit codes
+_RETCODE_HINTS = {
+	127: '"Command not found".  Make sure that all commands in %(realCmd)r '
+			'are in the PATH of fail2ban-server process '
+			'(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). '
+			'You may want to start '
+			'"fail2ban-server -f" separately, initiate it with '
+			'"fail2ban-client reload" in another shell session and observe if '
+			'additional informative error messages appear in the terminals.'
+	}
+
+# Dictionary to lookup signal name from number
+signame = dict((num, name)
+	for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))
+
+class Utils():
+	"""Utilities provide diverse static methods like executes OS shell commands, etc.
+	"""
+
+	DEFAULT_SLEEP_TIME = 0.1
+	DEFAULT_SLEEP_INTERVAL = 0.01
+
+
+	class Cache(dict):
+
+		def __init__(self, maxCount=1000, maxTime=60*60):
+			self.maxCount = maxCount
+			self.maxTime = maxTime
+
+		def get(self, k, defv=None):
+			v = dict.get(self, k)
+			if v: 
+				if v[1] > time.time():
+					return v[0]
+				del self[k]
+			return defv
+			
+		def set(self, k, v):
+			t = time.time()
+			# clean cache if max count reached:
+			if len(self) >= self.maxCount:
+				for (ck,cv) in self.items():
+					if cv[1] < t:
+						del self[ck]
+				# if still max count - remove any one:
+				if len(self) >= self.maxCount:
+					self.popitem()
+			self[k] = (v, t + self.maxTime)
+
+
+	@staticmethod
+	def setFBlockMode(fhandle, value):
+		flags = fcntl.fcntl(fhandle, fcntl.F_GETFL)
+		if not value:
+			flags |= os.O_NONBLOCK 
+		else:
+			flags &= ~os.O_NONBLOCK
+		fcntl.fcntl(fhandle, fcntl.F_SETFL, flags)
+		return flags
+
+	@staticmethod
+	def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True):
+		"""Executes a command.
+
+		Parameters
+		----------
+		realCmd : str
+			The command to execute.
+		timeout : int
+			The time out in seconds for the command.
+		shell : bool
+			If shell is True (default), the specified command (may be a string) will be 
+			executed through the shell.
+		output : bool
+			If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode)
+
+		Returns
+		-------
+		bool
+			True if the command succeeded.
+
+		Raises
+		------
+		OSError
+			If command fails to be executed.
+		RuntimeError
+			If command execution times out.
+		"""
+		stdout = stderr = None
+		retcode = None
+		if not callable(timeout):
+			stime = time.time()
+			timeout_expr = lambda: time.time() - stime <= timeout
+		else:
+			timeout_expr = timeout
+		try:
+			popen = subprocess.Popen(
+				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell,
+				preexec_fn=os.setsid  # so that killpg does not kill our process
+			)
+			retcode = popen.poll()
+			while retcode is None and timeout_expr():
+				time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
+				retcode = popen.poll()
+			if retcode is None:
+				logSys.error("%s -- timed out after %s seconds." %
+					(realCmd, timeout))
+				pgid = os.getpgid(popen.pid)
+				# if not tree - first try to terminate and then kill, otherwise - kill (-9) only:
+				os.killpg(pgid, signal.SIGTERM) # Terminate the process
+				time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
+				retcode = popen.poll()
+				#logSys.debug("%s -- terminated %s ", realCmd, retcode)
+				if retcode is None or tout_kill_tree: # Still going...
+					os.killpg(pgid, signal.SIGKILL) # Kill the process
+					time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
+					retcode = popen.poll()
+					#logSys.debug("%s -- killed %s ", realCmd, retcode)
+				if retcode is None and not Utils.pid_exists(pgid):
+					retcode = signal.SIGKILL
+		except OSError as e:
+			logSys.error("%s -- failed with %s" % (realCmd, e))
+
+		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
+		# if we need output (to return or to log it): 
+		if output or std_level >= logSys.getEffectiveLevel():
+			# if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
+			if popen.stdout:
+				try:
+					if retcode < 0:
+						Utils.setFBlockMode(popen.stdout, False)
+					stdout = popen.stdout.read()
+				except IOError as e:
+					logSys.error(" ... -- failed to read stdout %s", e)
+				if stdout is not None and stdout != '':
+					logSys.log(std_level, "%s -- stdout: %r", realCmd, stdout)
+				popen.stdout.close()
+			if popen.stderr:
+				try:
+					if retcode < 0:
+						Utils.setFBlockMode(popen.stderr, False)
+					stderr = popen.stderr.read()
+				except IOError as e:
+					logSys.error(" ... -- failed to read stderr %s", e)
+				if stderr is not None and stderr != '':
+					logSys.log(std_level, "%s -- stderr: %r", realCmd, stderr)
+				popen.stderr.close()
+
+		if retcode == 0:
+			logSys.debug("%s -- returned successfully", realCmd)
+			return True if not output else (True, stdout, stderr, retcode)
+		elif retcode is None:
+			logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid))
+		elif retcode < 0 or retcode > 128:
+			# dash would return negative while bash 128 + n
+			sigcode = -retcode if retcode < 0 else retcode - 128
+			logSys.error("%s -- killed with %s (return code: %s)" %
+				(realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode))
+		else:
+			msg = _RETCODE_HINTS.get(retcode, None)
+			logSys.error("%s -- returned %i" % (realCmd, retcode))
+			if msg:
+				logSys.info("HINT on %i: %s", retcode, msg % locals())
+		return False if not output else (False, stdout, stderr, retcode)
+	
+	@staticmethod
+	def wait_for(cond, timeout, interval=None):
+		"""Wait until condition expression `cond` is True, up to `timeout` sec
+		"""
+		ini = 1
+		while True:
+			ret = cond()
+			if ret:
+				return ret
+			if ini:
+				ini = stm = 0
+				time0 = time.time() + timeout
+				if not interval:
+					interval = Utils.DEFAULT_SLEEP_INTERVAL
+			if time.time() > time0:
+				break
+			stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME)
+			time.sleep(stm)
+		return ret
+
+	# Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
+	# under cc by-sa 3.0
+	if os.name == 'posix':
+		@staticmethod
+		def pid_exists(pid):
+			"""Check whether pid exists in the current process table."""
+			import errno
+			if pid < 0:
+				return False
+			try:
+				os.kill(pid, 0)
+			except OSError as e:
+				return e.errno == errno.EPERM
+			else:
+				return True
+	else:
+		@staticmethod
+		def pid_exists(pid):
+			import ctypes
+			kernel32 = ctypes.windll.kernel32
+			SYNCHRONIZE = 0x100000
+
+			process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid)
+			if process != 0:
+				kernel32.CloseHandle(process)
+				return True
+			else:
+				return False
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index a7f148b1..74594420 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -29,6 +29,8 @@ if sys.version_info >= (2,7):
 
 		def setUp(self):
 			"""Call before every test case."""
+			unittest.F2B.SkipIfNoNetwork()
+
 			self.jail = DummyJail()
 
 			self.jail.actions.add("test")
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 35ac2393..27442832 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -46,6 +46,8 @@ class SMTPActionTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		unittest.F2B.SkipIfNoNetwork()
+
 		self.jail = DummyJail()
 		pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")
 		pythonModuleName = os.path.basename(pythonModule.rstrip(".py"))
diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py
index 0ceb35d5..dff0d4c6 100644
--- a/fail2ban/tests/actionstestcase.py
+++ b/fail2ban/tests/actionstestcase.py
@@ -30,6 +30,7 @@ import tempfile
 
 from ..server.actions import Actions
 from ..server.ticket import FailTicket
+from ..server.utils import Utils
 from .dummyjail import DummyJail
 from .utils import LogCaptureTestCase
 
@@ -81,8 +82,7 @@ class ExecuteActions(LogCaptureTestCase):
 		self.defaultActions()
 		self.__actions.start()
 		with open(self.__tmpfilename) as f:
-			time.sleep(3)
-			self.assertEqual(f.read(),"ip start 64\n")
+			self.assertTrue( Utils.wait_for(lambda: (f.read() == "ip start 64\n"), 3) )
 
 		self.__actions.stop()
 		self.__actions.join()
@@ -97,8 +97,7 @@ class ExecuteActions(LogCaptureTestCase):
 		self.assertLogged("TestAction initialised")
 
 		self.__actions.start()
-		time.sleep(3)
-		self.assertLogged("TestAction action start")
+		self.assertTrue( Utils.wait_for(lambda: self._is_logged("TestAction action start"), 3) )
 
 		self.__actions.stop()
 		self.__actions.join()
@@ -135,8 +134,7 @@ class ExecuteActions(LogCaptureTestCase):
 				"action.d/action_errors.py"),
 			{})
 		self.__actions.start()
-		time.sleep(3)
-		self.assertLogged("Failed to start")
+		self.assertTrue( Utils.wait_for(lambda: self._is_logged("Failed to start"), 3) )
 		self.__actions.stop()
 		self.__actions.join()
 		self.assertLogged("Failed to stop")
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 289d8896..8c9b5ef1 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -25,10 +25,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 import os
-import time
 import tempfile
+import time
+import unittest
 
 from ..server.action import CommandAction, CallingMap
+from ..server.utils import Utils
 
 from .utils import LogCaptureTestCase
 from .utils import pid_exists
@@ -194,16 +196,17 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertLogged('HINT on 127: "Command not found"')
 
 	def testExecuteTimeout(self):
+		unittest.F2B.SkipIfFast()
 		stime = time.time()
 		# Should take a minute
-		self.assertFalse(CommandAction.executeCmd('sleep 60', timeout=2))
+		self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=1))
 		# give a test still 1 second, because system could be too busy
-		self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3)
+		self.assertTrue(time.time() >= stime + 1 and time.time() <= stime + 2)
 		self.assertLogged(
-			'sleep 60 -- timed out after 2 seconds',
-			'sleep 60 -- timed out after 3 seconds'
+			'sleep 30 -- timed out after 1 seconds',
+			'sleep 30 -- timed out after 2 seconds'
 		)
-		self.assertLogged('sleep 60 -- killed with SIGTERM')
+		self.assertLogged('sleep 30 -- killed with SIGTERM')
 
 	def testExecuteTimeoutWithNastyChildren(self):
 		# temporary file for a nasty kid shell script
@@ -215,29 +218,53 @@ class CommandActionTest(LogCaptureTestCase):
 
 		echo "$$" > %s.pid
 		echo "my pid $$ . sleeping lo-o-o-ong"
-		sleep 10000
+		sleep 30
 		""" % tmpFilename)
+		stime = 0
+
+		# timeout as long as pid-file was not created, but max 5 seconds
+		def getnasty_tout():
+			return (
+				getnastypid() is None
+				and time.time() - stime <= 5
+			)
 
 		def getnastypid():
-			with open(tmpFilename + '.pid') as f:
-				return int(f.read())
+			cpid = None
+			if os.path.isfile(tmpFilename + '.pid'):
+				with open(tmpFilename + '.pid') as f:
+					try:
+						cpid = int(f.read())
+					except ValueError:
+						pass
+			return cpid
 
 		# First test if can kill the bastard
+		stime = time.time()
 		self.assertFalse(CommandAction.executeCmd(
-		                 'bash %s' % tmpFilename, timeout=.1))
+			'bash %s' % tmpFilename, timeout=getnasty_tout))
+		# Wait up to 3 seconds, the child got killed
+		cpid = getnastypid()
 		# Verify that the process itself got killed
-		self.assertFalse(pid_exists(getnastypid()))  # process should have been killed
+		self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3))  # process should have been killed
+		self.assertLogged('my pid ')
 		self.assertLogged('timed out')
-		self.assertLogged('killed with SIGTERM')
+		self.assertLogged('killed with SIGTERM', 
+		                  'killed with SIGKILL')
+		os.unlink(tmpFilename + '.pid')
 
 		# A bit evolved case even though, previous test already tests killing children processes
+		stime = time.time()
 		self.assertFalse(CommandAction.executeCmd(
-			'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2))
+			'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=getnasty_tout))
+		# Wait up to 3 seconds, the child got killed
+		cpid = getnastypid()
 		# Verify that the process itself got killed
-		self.assertFalse(pid_exists(getnastypid()))
+		self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3))
+		self.assertLogged('my pid ')
 		self.assertLogged('timed out')
-		self.assertLogged('killed with SIGTERM')
-
+		self.assertLogged('killed with SIGTERM', 
+		                  'killed with SIGKILL')
 		os.unlink(tmpFilename)
 		os.unlink(tmpFilename + '.pid')
 
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 09d2411e..a2d399b3 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -60,6 +60,7 @@ class AddFailure(unittest.TestCase):
 class StatusExtendedCymruInfo(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
+		unittest.F2B.SkipIfNoNetwork()
 		self.__ban_ip = "93.184.216.34"
 		self.__asn = "15133"
 		self.__country = "EU"
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 083be2b2..20baa847 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -35,7 +35,12 @@ from ..server.ticket import FailTicket
 from ..server.actions import Actions
 from .dummyjail import DummyJail
 try:
-	from ..server.database import Fail2BanDb
+	from ..server.database import Fail2BanDb as Fail2BanDb
+	# because of tests performance use memory instead of file:
+	def TestFail2BanDb(filename):
+		if unittest.F2B.fast:
+			return Fail2BanDb(':memory:')
+		return Fail2BanDb(filename)
 except ImportError:
 	Fail2BanDb = None
 from .utils import LogCaptureTestCase
@@ -55,7 +60,7 @@ class DatabaseTest(LogCaptureTestCase):
 		elif Fail2BanDb is None:
 			return
 		_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
-		self.db = Fail2BanDb(self.dbFilename)
+		self.db = TestFail2BanDb(self.dbFilename)
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -66,7 +71,7 @@ class DatabaseTest(LogCaptureTestCase):
 		os.remove(self.dbFilename)
 
 	def testGetFilename(self):
-		if Fail2BanDb is None: # pragma: no cover
+		if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover
 			return
 		self.assertEqual(self.dbFilename, self.db.filename)
 
@@ -88,7 +93,7 @@ class DatabaseTest(LogCaptureTestCase):
 			"/this/path/should/not/exist")
 
 	def testCreateAndReconnect(self):
-		if Fail2BanDb is None: # pragma: no cover
+		if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover
 			return
 		self.testAddJail()
 		# Reconnect...
diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index 6be571d3..bade2f12 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -39,28 +39,27 @@ class DummyJail(Jail, object):
 		self.__actions = Actions(self)
 
 	def __len__(self):
-		try:
-			self.lock.acquire()
+		with self.lock:
 			return len(self.queue)
-		finally:
-			self.lock.release()
+
+	def isEmpty(self):
+		with self.lock:
+			return not self.queue
+
+	def isFilled(self):
+		with self.lock:
+			return bool(self.queue)
 
 	def putFailTicket(self, ticket):
-		try:
-			self.lock.acquire()
+		with self.lock:
 			self.queue.append(ticket)
-		finally:
-			self.lock.release()
 
 	def getFailTicket(self):
-		try:
-			self.lock.acquire()
+		with self.lock:
 			try:
 				return self.queue.pop()
 			except IndexError:
 				return False
-		finally:
-			self.lock.release()
 
 	@property
 	def name(self):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 7a35bdd9..59512530 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -41,6 +41,7 @@ from ..server.filterpoll import FilterPoll
 from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils
 from ..server.failmanager import FailManagerEmpty
 from ..server.mytime import MyTime
+from ..server.utils import Utils
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
 from .dummyjail import DummyJail
 
@@ -162,7 +163,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line
 		# Opened earlier, therefore must close it
 		fin.close()
 	# to give other threads possibly some time to crunch
-	time.sleep(0.1)
+	time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
 	return fout
 
 
@@ -299,6 +300,11 @@ class IgnoreIP(LogCaptureTestCase):
 
 class IgnoreIPDNS(IgnoreIP):
 
+	def setUp(self):
+		"""Call before every test case."""
+		unittest.F2B.SkipIfNoNetwork()
+		IgnoreIP.setUp(self)
+
 	def testIgnoreIPDNSOK(self):
 		self.filter.addIgnoreIP("www.epfl.ch")
 		self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12"))
@@ -425,16 +431,11 @@ class LogFileMonitor(LogCaptureTestCase):
 	def isModified(self, delay=2.):
 		"""Wait up to `delay` sec to assure that it was modified or not
 		"""
-		time0 = time.time()
-		while time.time() < time0 + delay:
-			if self.filter.isModified(self.name):
-				return True
-			time.sleep(0.1)
-		return False
+		return Utils.wait_for(lambda: self.filter.isModified(self.name), delay)
 
 	def notModified(self):
 		# shorter wait time for not modified status
-		return not self.isModified(0.4)
+		return not self.isModified(4*Utils.DEFAULT_SLEEP_TIME)
 
 	def testUnaccessibleLogFile(self):
 		os.chmod(self.name, 0)
@@ -583,26 +584,21 @@ def get_monitor_failures_testcase(Filter_):
 			#time.sleep(0.2)			  # Give FS time to ack the removal
 			pass
 
-		def isFilled(self, delay=2.):
+		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			time0 = time.time()
-			while time.time() < time0 + delay:
-				if len(self.jail):
-					return True
-				time.sleep(0.1)
-			return False
+			return Utils.wait_for(lambda: self.jail.isFilled(), delay)
 
 		def _sleep_4_poll(self):
 			# Since FilterPoll relies on time stamps and some
 			# actions might be happening too fast in the tests,
 			# sleep a bit to guarantee reliable time stamps
 			if isinstance(self.filter, FilterPoll):
-				mtimesleep()
+				Utils.wait_for(lambda: self.filter.is_alive(), 4*Utils.DEFAULT_SLEEP_TIME)
 
-		def isEmpty(self, delay=0.4):
+		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
 			# shorter wait time for not modified status
-			return not self.isFilled(delay)
+			return Utils.wait_for(lambda: self.jail.isEmpty(), delay)
 
 		def assert_correct_last_attempt(self, failures, count=None):
 			self.assertTrue(self.isFilled(20)) # give Filter a chance to react
@@ -657,10 +653,11 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  n=14, mode='w')
 			# Poll might need more time
-			self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2),
+			self.assertTrue(self.isEmpty(min(4, 100 * Utils.DEFAULT_SLEEP_TIME)),
 							"Queue must be empty but it is not: %s."
 							% (', '.join([str(x) for x in self.jail.queue])))
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+			Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, 50 * Utils.DEFAULT_SLEEP_TIME)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 2)
 
 			# move aside, but leaving the handle still open...
@@ -685,7 +682,7 @@ def get_monitor_failures_testcase(Filter_):
 
 			if interim_kill:
 				_killfile(None, self.name)
-				time.sleep(0.2)				  # let them know
+				time.sleep(Utils.DEFAULT_SLEEP_TIME)				  # let them know
 
 			# now create a new one to override old one
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new',
@@ -732,7 +729,7 @@ def get_monitor_failures_testcase(Filter_):
 
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
 			# so we should get no more failures detected
-			self.assertTrue(self.isEmpty(2))
+			self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME))
 
 			# but then if we add it back again
 			self.filter.addLogPath(self.name)
@@ -789,19 +786,14 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			return "MonitorJournalFailures%s(%s)" \
 			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
 
-		def isFilled(self, delay=2.):
+		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			time0 = time.time()
-			while time.time() < time0 + delay:
-				if len(self.jail):
-					return True
-				time.sleep(0.1)
-			return False
+			return Utils.wait_for(lambda: self.jail.isFilled(), delay)
 
-		def isEmpty(self, delay=0.4):
+		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
 			# shorter wait time for not modified status
-			return not self.isFilled(delay)
+			return Utils.wait_for(lambda: self.jail.isEmpty(), delay)
 
 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
@@ -860,7 +852,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, n=5, skip=5)
 			# so we should get no more failures detected
-			self.assertTrue(self.isEmpty(2))
+			self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME))
 
 			# but then if we add it back again
 			self.filter.addJournalMatch([
@@ -905,6 +897,16 @@ class GetFailures(LogCaptureTestCase):
 		tearDownMyTime()
 		LogCaptureTestCase.tearDown(self)
 
+	def testFilterAPI(self):
+		self.assertEqual(self.filter.getLogs(), [])
+		self.assertEqual(self.filter.getLogCount(), 0)
+		self.filter.addLogPath(GetFailures.FILENAME_01, tail=True)
+		self.assertEqual(self.filter.getLogCount(), 1)
+		self.assertEqual(self.filter.getLogPaths(), [GetFailures.FILENAME_01])
+		self.filter.addLogPath(GetFailures.FILENAME_02, tail=True)
+		self.assertEqual(self.filter.getLogCount(), 2)
+		self.assertEqual(sorted(self.filter.getLogPaths()), sorted([GetFailures.FILENAME_01, GetFailures.FILENAME_02]))
+
 	def testTail(self):
 		# There must be no containters registered, otherwise [-1] indexing would be wrong
 		self.assertEqual(self.filter.getLogs(), [])
@@ -1025,6 +1027,7 @@ class GetFailures(LogCaptureTestCase):
 			_killfile(fout, fname)
 
 	def testGetFailuresUseDNS(self):
+		unittest.F2B.SkipIfNoNetwork()
 		# We should still catch failures with usedns = no ;-)
 		output_yes = ('93.184.216.34', 2, 1124013539.0,
 					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
@@ -1126,6 +1129,10 @@ class GetFailures(LogCaptureTestCase):
 
 class DNSUtilsTests(unittest.TestCase):
 
+	def setUp(self):
+		"""Call before every test case."""
+		unittest.F2B.SkipIfNoNetwork()
+
 	def testUseDns(self):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
@@ -1151,6 +1158,7 @@ class DNSUtilsTests(unittest.TestCase):
 	def testIpToName(self):
 		res = DNSUtils.ipToName('8.8.4.4')
 		self.assertEqual(res, 'google-public-dns-b.google.com')
+		unittest.F2B.SkipIfNoNetwork()
 		# invalid ip (TEST-NET-1 according to RFC 5737)
 		res = DNSUtils.ipToName('192.0.2.0')
 		self.assertEqual(res, None)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index a7d7adef..d547b79a 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -36,6 +36,7 @@ from ..server.failregex import Regex, FailRegex, RegexException
 from ..server.server import Server
 from ..server.jail import Jail
 from ..server.jailthread import JailThread
+from ..server.utils import Utils
 from .utils import LogCaptureTestCase
 from ..helpers import getLogger
 from .. import version
@@ -74,14 +75,14 @@ class TransmitterBase(unittest.TestCase):
 		"""Call after every test case."""
 		self.server.quit()
 
-	def setGetTest(self, cmd, inValue, outValue=None, outCode=0, jail=None, repr_=False):
+	def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False):
 		setCmd = ["set", cmd, inValue]
 		getCmd = ["get", cmd]
 		if jail is not None:
 			setCmd.insert(1, jail)
 			getCmd.insert(1, jail)
 
-		if outValue is None:
+		if outValue == (None,):
 			outValue = inValue
 
 		def v(x):
@@ -161,15 +162,21 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(self.transm.proceed(["version"]), (0, version.version))
 
 	def testSleep(self):
-		t0 = time.time()
-		self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None))
-		t1 = time.time()
-		# Approx 1 second delay but not faster
-		dt = t1 - t0
-		self.assertTrue(0.99 < dt < 1.1, msg="Sleep was %g sec" % dt)
+		if not unittest.F2B.fast:
+			t0 = time.time()
+			self.assertEqual(self.transm.proceed(["sleep", "0.1"]), (0, None))
+			t1 = time.time()
+			# Approx 0.1 second delay but not faster
+			dt = t1 - t0
+			self.assertTrue(0.09 < dt < 0.2, msg="Sleep was %g sec" % dt)
+		else: # pragma: no cover
+			self.assertEqual(self.transm.proceed(["sleep", "0.0001"]), (0, None))
 
 	def testDatabase(self):
-		tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_")
+		if not unittest.F2B.fast:
+			tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_")
+		else: # pragma: no cover
+			tmpFilename = ':memory:'
 		# Jails present, can't change database
 		self.setGetTestNOK("dbfile", tmpFilename)
 		self.server.delJail(self.jailName)
@@ -201,8 +208,9 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(self.transm.proceed(
 			["set", "dbfile", "None"]),
 			(0, None))
-		os.close(tmp)
-		os.unlink(tmpFilename)
+		if not unittest.F2B.fast:
+			os.close(tmp)
+			os.unlink(tmpFilename)
 
 	def testAddJail(self):
 		jail2 = "TestJail2"
@@ -225,7 +233,11 @@ class Transmitter(TransmitterBase):
 	def testStartStopJail(self):
 		self.assertEqual(
 			self.transm.proceed(["start", self.jailName]), (0, None))
-		time.sleep(1)
+		time.sleep(Utils.DEFAULT_SLEEP_TIME)
+		# wait until not started (3 seconds as long as any RuntimeError, ex.: RuntimeError('cannot join thread before it is started',)):
+		self.assertTrue( Utils.wait_for(
+			lambda: self.server.is_alive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 
+			3) )
 		self.assertEqual(
 			self.transm.proceed(["stop", self.jailName]), (0, None))
 		self.assertTrue(self.jailName not in self.server._Server__jails)
@@ -239,9 +251,12 @@ class Transmitter(TransmitterBase):
 		# yoh: workaround for gh-146.  I still think that there is some
 		#      race condition and missing locking somewhere, but for now
 		#      giving it a small delay reliably helps to proceed with tests
-		time.sleep(0.1)
+		time.sleep(Utils.DEFAULT_SLEEP_TIME)
+		self.assertTrue( Utils.wait_for(
+			lambda: self.server.is_alive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 
+			3) )
 		self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None))
-		time.sleep(1)
+		self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) )
 		self.assertTrue(self.jailName not in self.server._Server__jails)
 		self.assertTrue("TestJail2" not in self.server._Server__jails)
 
@@ -297,11 +312,11 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]),
 			(0, "127.0.0.1"))
-		time.sleep(1) # Give chance to ban
+		time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "banip", "Badger"]),
 			(0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done?
-		time.sleep(1) # Give chance to ban
+		time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban
 		# Unban IP
 		self.assertEqual(
 			self.transm.proceed(
diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py
index 8eeb7b51..a9408fde 100644
--- a/fail2ban/tests/sockettestcase.py
+++ b/fail2ban/tests/sockettestcase.py
@@ -33,6 +33,7 @@ import unittest
 
 from .. import protocol
 from ..server.asyncserver import AsyncServer, AsyncServerException
+from ..server.utils import Utils
 from ..client.csocket import CSocket
 
 
@@ -54,14 +55,20 @@ class Socket(unittest.TestCase):
 		"""Test transmitter proceed method which just returns first arg"""
 		return message
 
+	def _serverSocket(self):
+		try:
+			return CSocket(self.sock_name)
+		except Exception as e:
+			return None
+
 	def testSocket(self):
 		serverThread = threading.Thread(
 			target=self.server.start, args=(self.sock_name, False))
 		serverThread.daemon = True
 		serverThread.start()
-		time.sleep(1)
+		time.sleep(Utils.DEFAULT_SLEEP_TIME)
 
-		client = CSocket(self.sock_name)
+		client = Utils.wait_for(self._serverSocket, 2)
 		testMessage = ["A", "test", "message"]
 		self.assertEqual(client.send(testMessage), testMessage)
 
@@ -71,7 +78,7 @@ class Socket(unittest.TestCase):
 		client.close()
 
 		self.server.stop()
-		serverThread.join(1)
+		serverThread.join(Utils.DEFAULT_SLEEP_TIME)
 		self.assertFalse(os.path.exists(self.sock_name))
 
 	def testSocketForce(self):
@@ -85,10 +92,10 @@ class Socket(unittest.TestCase):
 			target=self.server.start, args=(self.sock_name, True))
 		serverThread.daemon = True
 		serverThread.start()
-		time.sleep(1)
+		time.sleep(Utils.DEFAULT_SLEEP_TIME)
 
 		self.server.stop()
-		serverThread.join(1)
+		serverThread.join(Utils.DEFAULT_SLEEP_TIME)
 		self.assertFalse(os.path.exists(self.sock_name))
 
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index dff3faae..a66325d8 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -30,8 +30,10 @@ import time
 import unittest
 from StringIO import StringIO
 
-from ..server.mytime import MyTime
 from ..helpers import getLogger
+from ..server.filter import DNSUtils
+from ..server.mytime import MyTime
+from ..server.utils import Utils
 
 logSys = getLogger(__name__)
 
@@ -45,6 +47,43 @@ if not CONFIG_DIR:
 		CONFIG_DIR = '/etc/fail2ban'
 
 
+class F2B():
+	def __init__(self, fast=False, no_network=False):
+		self.fast=fast
+		self.no_network=no_network
+	def SkipIfFast(self):
+		pass
+	def SkipIfNoNetwork(self):
+		pass
+
+
+def initTests(opts):
+	if opts: # pragma: no cover
+		unittest.F2B = F2B(opts.fast, opts.no_network)
+	else:
+		unittest.F2B = F2B()
+	# --fast :
+	if unittest.F2B.fast: # pragma: no cover
+		# prevent long sleeping during test cases...
+		Utils.DEFAULT_SLEEP_TIME = 0.0025
+		Utils.DEFAULT_SLEEP_INTERVAL = 0.0005
+		def F2B_SkipIfFast():
+			raise unittest.SkipTest('Skip test because of "--fast"')
+		unittest.F2B.SkipIfFast = F2B_SkipIfFast
+	else:
+		# sleep intervals are large - use replacement for sleep to check time to sleep:
+		_org_sleep = time.sleep
+		def _new_sleep(v):
+			if (v > Utils.DEFAULT_SLEEP_TIME):
+				raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v)
+			_org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME))
+		time.sleep = _new_sleep
+	# --no-network :
+	if unittest.F2B.no_network: # pragma: no cover
+		def F2B_SkipIfNoNetwork():
+			raise unittest.SkipTest('Skip test because of "--no-network"')
+		unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork
+
 def mtimesleep():
 	# no sleep now should be necessary since polling tracks now not only
 	# mtime but also ino and size
@@ -70,7 +109,8 @@ def tearDownMyTime():
 	MyTime.myTime = None
 
 
-def gatherTests(regexps=None, no_network=False):
+def gatherTests(regexps=None, opts=None):
+	initTests(opts)
 	# Import all the test cases here instead of a module level to
 	# avoid circular imports
 	from . import banmanagertestcase
@@ -143,10 +183,10 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(filtertestcase.LogFile))
 	tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor))
 	tests.addTest(unittest.makeSuite(filtertestcase.LogFileFilterPoll))
-	if not no_network:
-		tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS))
-		tests.addTest(unittest.makeSuite(filtertestcase.GetFailures))
-		tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests))
+	# each test case class self will check no network, and skip it (we see it in log)
+	tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS))
+	tests.addTest(unittest.makeSuite(filtertestcase.GetFailures))
+	tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests))
 	tests.addTest(unittest.makeSuite(filtertestcase.JailTests))
 
 	# DateDetector
@@ -165,9 +205,6 @@ def gatherTests(regexps=None, no_network=False):
 	for file_ in os.listdir(
 		os.path.abspath(os.path.dirname(action_d.__file__))):
 		if file_.startswith("test_") and file_.endswith(".py"):
-			if no_network and file_ in ['test_badips.py','test_smtp.py']: #pragma: no cover
-				# Test required network
-				continue
 			tests.addTest(testloader.loadTestsFromName(
 				"%s.%s" % (action_d.__name__, os.path.splitext(file_)[0])))
 
@@ -182,6 +219,9 @@ def gatherTests(regexps=None, no_network=False):
 	# yoh: Since I do not know better way for parametric tests
 	#      with good old unittest
 	try:
+		# because gamin can be very slow on some platforms (and can produce many failures 
+		# with fast sleep interval) - skip it by fast run:
+		unittest.F2B.SkipIfFast()
 		from ..server.filtergamin import FilterGamin
 		filters.append(FilterGamin)
 	except Exception, e: # pragma: no cover
@@ -276,29 +316,4 @@ class LogCaptureTestCase(unittest.TestCase):
 	def printLog(self):
 		print(self._log.getvalue())
 
-# Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
-# under cc by-sa 3.0
-if os.name == 'posix':
-	def pid_exists(pid):
-		"""Check whether pid exists in the current process table."""
-		import errno
-		if pid < 0:
-			return False
-		try:
-			os.kill(pid, 0)
-		except OSError as e:
-			return e.errno == errno.EPERM
-		else:
-			return True
-else:
-	def pid_exists(pid):
-		import ctypes
-		kernel32 = ctypes.windll.kernel32
-		SYNCHRONIZE = 0x100000
-
-		process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid)
-		if process != 0:
-			kernel32.CloseHandle(process)
-			return True
-		else:
-			return False
+pid_exists = Utils.pid_exists

From 78eb4d95eb04c6aeb8b7afa1382a6a48f9ad584a Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 16 Jul 2015 17:14:10 -0400
Subject: [PATCH 011/623] RF: is_alive -> isAlive  to stay consistent in
 camelCasing

---
 fail2ban/server/jail.py          | 6 +++---
 fail2ban/server/server.py        | 8 ++++----
 fail2ban/tests/dummyjail.py      | 2 +-
 fail2ban/tests/filtertestcase.py | 2 +-
 fail2ban/tests/servertestcase.py | 4 ++--
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index c5f2e9cd..cf16737a 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -229,7 +229,7 @@ class Jail:
 		self.actions.join()
 		logSys.info("Jail '%s' stopped" % self.name)
 
-	def is_alive(self):
-		"""Check jail "is_alive" by checking filter and actions threads.
+	def isAlive(self):
+		"""Check jail "isAlive" by checking filter and actions threads.
 		"""
-		return self.filter.is_alive() or self.actions.is_alive()
+		return self.filter.isAlive() or self.actions.isAlive()
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 1b335d02..cdfb4465 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -157,7 +157,7 @@ class Server:
 	def startJail(self, name):
 		try:
 			self.__lock.acquire()
-			if not self.__jails[name].is_alive():
+			if not self.__jails[name].isAlive():
 				self.__jails[name].start()
 		finally:
 			self.__lock.release()
@@ -166,7 +166,7 @@ class Server:
 		logSys.debug("Stopping jail %s" % name)
 		try:
 			self.__lock.acquire()
-			if self.__jails[name].is_alive():
+			if self.__jails[name].isAlive():
 				self.__jails[name].stop()
 				self.delJail(name)
 		finally:
@@ -323,12 +323,12 @@ class Server:
 	def getBanTime(self, name):
 		return self.__jails[name].actions.getBanTime()
 	
-	def is_alive(self, jailnum=None):
+	def isAlive(self, jailnum=None):
 		if jailnum is not None and len(self.__jails) != jailnum:
 			return 0
 		for j in self.__jails:
 			j = self.__jails[j]
-			if not j.is_alive():
+			if not j.isAlive():
 				return 0
 		return 1
 
diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index bade2f12..33a102f2 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -85,5 +85,5 @@ class DummyJail(Jail, object):
 	def actions(self):
 		return self.__actions;
 
-	def is_alive(self):
+	def isAlive(self):
 		return True;
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 59512530..044e0346 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -594,7 +594,7 @@ def get_monitor_failures_testcase(Filter_):
 			# actions might be happening too fast in the tests,
 			# sleep a bit to guarantee reliable time stamps
 			if isinstance(self.filter, FilterPoll):
-				Utils.wait_for(lambda: self.filter.is_alive(), 4*Utils.DEFAULT_SLEEP_TIME)
+				Utils.wait_for(lambda: self.filter.isAlive(), 4*Utils.DEFAULT_SLEEP_TIME)
 
 		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
 			# shorter wait time for not modified status
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d547b79a..eedc8c94 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -236,7 +236,7 @@ class Transmitter(TransmitterBase):
 		time.sleep(Utils.DEFAULT_SLEEP_TIME)
 		# wait until not started (3 seconds as long as any RuntimeError, ex.: RuntimeError('cannot join thread before it is started',)):
 		self.assertTrue( Utils.wait_for(
-			lambda: self.server.is_alive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 
+			lambda: self.server.isAlive(1) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError),
 			3) )
 		self.assertEqual(
 			self.transm.proceed(["stop", self.jailName]), (0, None))
@@ -253,7 +253,7 @@ class Transmitter(TransmitterBase):
 		#      giving it a small delay reliably helps to proceed with tests
 		time.sleep(Utils.DEFAULT_SLEEP_TIME)
 		self.assertTrue( Utils.wait_for(
-			lambda: self.server.is_alive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError), 
+			lambda: self.server.isAlive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError),
 			3) )
 		self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None))
 		self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) )

From e2a68bb02de740fb3a4d095d0af323f289e112fd Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 16 Jul 2015 17:15:11 -0400
Subject: [PATCH 012/623] RF: make Jail new style class, avoiding multiple
 inheritance with object

---
 fail2ban/server/jail.py     | 2 +-
 fail2ban/tests/dummyjail.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index cf16737a..6345d300 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -33,7 +33,7 @@ from ..helpers import getLogger
 logSys = getLogger(__name__)
 
 
-class Jail:
+class Jail(object):
 	"""Fail2Ban jail, which manages a filter and associated actions.
 
 	The class handles the initialisation of a filter, and actions. It's
diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index 33a102f2..19f97f4e 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -28,7 +28,7 @@ from ..server.jail import Jail
 from ..server.actions import Actions
 
 
-class DummyJail(Jail, object):
+class DummyJail(Jail):
 	"""A simple 'jail' to suck in all the tickets generated by Filter's
 	"""
 	def __init__(self, backend=None):

From 0e61b7586a5e8ca4bb769e14066af403d55d5bd0 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 16 Jul 2015 17:22:36 -0400
Subject: [PATCH 013/623] RF: use descriptive loop variable (oldticket) and
 matching associated variable suffixes (old)

---
 fail2ban/server/banmanager.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index a958a5c8..0ee028ef 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -263,19 +263,19 @@ class BanManager:
 		try:
 			self.__lock.acquire()
 			# check already banned
-			for i in self.__banList:
-				if ticket.getIP() == i.getIP():
+			for oldticket in self.__banList:
+				if ticket.getIP() == oldticket.getIP():
 					# if already permanent
-					btorg, torg = i.getBanTime(self.__banTime), i.getTime()
-					if btorg == -1:
+					btold, told = oldticket.getBanTime(self.__banTime), oldticket.getTime()
+					if btold == -1:
 						return False
 					# if given time is less than already banned time
 					btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
-					if btnew != -1 and tnew + btnew <= torg + btorg:
+					if btnew != -1 and tnew + btnew <= told + btold:
 						return False
 					# we have longest ban - set new (increment) ban time
-					i.setTime(tnew)
-					i.setBanTime(btnew)
+					oldticket.setTime(tnew)
+					oldticket.setBanTime(btnew)
 					return False
 			# not yet banned - add new
 			self.__banList.append(ticket)

From 6faffe3201ee34d98c36c985a35f2c3810a40aec Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 17 Jul 2015 18:34:13 +0200
Subject: [PATCH 014/623] test cases extended, code review (+ python 3.x
 compatibility); database test cases extended - enable deleted (disabled) jail
 in addJail;

---
 MANIFEST                             |   1 +
 fail2ban/server/database.py          |   8 +-
 fail2ban/server/filter.py            |   4 +-
 fail2ban/server/server.py            |   2 +-
 fail2ban/server/ticket.py            |  21 ++--
 fail2ban/server/utils.py             |   5 +-
 fail2ban/tests/banmanagertestcase.py |  51 +++++++-
 fail2ban/tests/databasetestcase.py   |  21 +++-
 fail2ban/tests/filtertestcase.py     |  56 ++++++++-
 fail2ban/tests/tickettestcase.py     | 176 +++++++++++++++++++++++++++
 fail2ban/tests/utils.py              |   4 +
 11 files changed, 325 insertions(+), 24 deletions(-)
 create mode 100644 fail2ban/tests/tickettestcase.py

diff --git a/MANIFEST b/MANIFEST
index 7306cc41..fb70bb4b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -331,6 +331,7 @@ fail2ban/tests/misctestcase.py
 fail2ban/tests/samplestestcase.py
 fail2ban/tests/servertestcase.py
 fail2ban/tests/sockettestcase.py
+fail2ban/tests/tickettestcase.py
 fail2ban/tests/utils.py
 fail2ban/version.py
 files/bash-completion
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index c6e8c95c..3b419ed3 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -302,7 +302,7 @@ class Fail2BanDb(object):
 		cur.execute("UPDATE jails SET enabled=0")
 
 	@commitandrollback
-	def getJailNames(self, cur):
+	def getJailNames(self, cur, enabled=None):
 		"""Get name of jails in database.
 
 		Currently only used for testing purposes.
@@ -312,7 +312,11 @@ class Fail2BanDb(object):
 		set
 			Set of jail names.
 		"""
-		cur.execute("SELECT name FROM jails")
+		if enabled is None:
+			cur.execute("SELECT name FROM jails")
+		else:
+			cur.execute("SELECT name FROM jails WHERE enabled=%s" %
+				(int(enabled),))
 		return set(row[0] for row in cur.fetchmany())
 
 	@commitandrollback
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 0a2a9949..e5cf648c 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -967,8 +967,8 @@ class DNSUtils:
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 
 	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=60*60)
-	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=60*60)
+	CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
 
 	@staticmethod
 	def dnsToIp(dns):
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index cdfb4465..86197bc5 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -44,7 +44,7 @@ logSys = getLogger(__name__)
 
 try:
 	from .database import Fail2BanDb
-except ImportError:
+except ImportError: # pragma: no cover
 	# Dont print error here, as database may not even be used
 	Fail2BanDb = None
 
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 49ebf9ea..e856b66d 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -146,16 +146,17 @@ class Ticket:
 		# return default if not exists:
 		if not self._data:
 			return default
-		# return filtered by lambda/function:
-		if callable(key):
-			# todo: if support >= 2.7 only:
-			# return {k:v for k,v in self._data.iteritems() if key(k)}
-			return dict([(k,v) for k,v in self._data.iteritems() if key(k)])
-		# return filtered by keys:
-		if hasattr(key, '__iter__'):
-			# todo: if support >= 2.7 only:
-			# return {k:v for k,v in self._data.iteritems() if k in key}
-			return dict([(k,v) for k,v in self._data.iteritems() if k in key])
+		if not isinstance(key,(str,unicode,type(None),int,float,bool,complex)):
+			# return filtered by lambda/function:
+			if callable(key):
+				# todo: if support >= 2.7 only:
+				# return {k:v for k,v in self._data.iteritems() if key(k)}
+				return dict([(k,v) for k,v in self._data.iteritems() if key(k)])
+			# return filtered by keys:
+			if hasattr(key, '__iter__'):
+				# todo: if support >= 2.7 only:
+				# return {k:v for k,v in self._data.iteritems() if k in key}
+				return dict([(k,v) for k,v in self._data.iteritems() if k in key])
 		# return single value of data:
 		return self._data.get(key, default)
 
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 7e69ddca..262b303d 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -52,7 +52,10 @@ class Utils():
 
 	class Cache(dict):
 
-		def __init__(self, maxCount=1000, maxTime=60*60):
+		def __init__(self, *args, **kwargs):
+			self.setOptions(*args, **kwargs)
+
+		def setOptions(self, maxCount=1000, maxTime=60):
 			self.maxCount = maxCount
 			self.maxTime = maxTime
 
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index a2d399b3..9e865a1b 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -35,27 +35,74 @@ class AddFailure(unittest.TestCase):
 		"""Call before every test case."""
 		self.__ticket = BanTicket('193.168.0.128', 1167605999.0)
 		self.__banManager = BanManager()
-		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 
 	def tearDown(self):
 		"""Call after every test case."""
 		pass
 
 	def testAdd(self):
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 		self.assertEqual(self.__banManager.size(), 1)
-
+		self.assertEqual(self.__banManager.getBanTotal(), 1)
+		self.__banManager.setBanTotal(0)
+		self.assertEqual(self.__banManager.getBanTotal(), 0)
+	
 	def testAddDuplicate(self):
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 		self.assertFalse(self.__banManager.addBanTicket(self.__ticket))
 		self.assertEqual(self.__banManager.size(), 1)
 
+	def testAddDuplicateWithTime(self):
+		# add again a duplicate :
+		#   1) with newer start time and the same ban time
+		#   2) with same start time and longer ban time
+    #   3) with permanent ban time (-1)
+		for tnew, btnew in (
+			(1167605999.0 + 100, None),
+			(1167605999.0,       24*60*60),
+			(1167605999.0,       -1),
+		):
+			ticket1 = BanTicket('193.168.0.128', 1167605999.0)
+			ticket2 = BanTicket('193.168.0.128', tnew)
+			if btnew is not None:
+				ticket2.setBanTime(btnew)
+			self.assertTrue(self.__banManager.addBanTicket(ticket1))
+			self.assertFalse(self.__banManager.addBanTicket(ticket2))
+			self.assertEqual(self.__banManager.size(), 1)
+			# pop ticket and check it was prolonged :
+			banticket = self.__banManager.getTicketByIP(ticket2.getIP())
+			self.assertEqual(banticket.getTime(), ticket2.getTime())
+			self.assertEqual(banticket.getTime(), ticket2.getTime())
+			self.assertEqual(banticket.getBanTime(), ticket2.getBanTime(self.__banManager.getBanTime()))
+
 	def testInListOK(self):
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 		ticket = BanTicket('193.168.0.128', 1167605999.0)
 		self.assertTrue(self.__banManager._inBanList(ticket))
 
 	def testInListNOK(self):
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 		ticket = BanTicket('111.111.1.111', 1167605999.0)
 		self.assertFalse(self.__banManager._inBanList(ticket))
 
+	def testUnban(self):
+		btime = self.__banManager.getBanTime()
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
+		self.assertTrue(self.__banManager._inBanList(self.__ticket))
+		self.assertEqual(self.__banManager.unBanList(self.__ticket.getTime() + btime + 1), [self.__ticket])
+		self.assertEqual(self.__banManager.size(), 0)
+
+	def testUnbanPermanent(self):
+		btime = self.__banManager.getBanTime()
+		self.__banManager.setBanTime(-1)
+		try:
+			self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
+			self.assertTrue(self.__banManager._inBanList(self.__ticket))
+			self.assertEqual(self.__banManager.unBanList(self.__ticket.getTime() + btime + 1), [])
+			self.assertEqual(self.__banManager.size(), 1)
+		finally:
+			self.__banManager.setBanTime(btime)
+
 
 class StatusExtendedCymruInfo(unittest.TestCase):
 	def setUp(self):
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 20baa847..3f0e4c10 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -124,7 +124,7 @@ class DatabaseTest(LogCaptureTestCase):
 		self.jail = DummyJail()
 		self.db.addJail(self.jail)
 		self.assertTrue(
-			self.jail.name in self.db.getJailNames(),
+			self.jail.name in self.db.getJailNames(True),
 			"Jail not added to database")
 
 	def testAddLog(self):
@@ -332,6 +332,25 @@ class DatabaseTest(LogCaptureTestCase):
 		actions._Actions__checkBan()
 		self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True))
 
+	def testDelAndAddJail(self):
+		self.testAddJail() # Add jail
+		# Delete jail (just disabled it):
+		self.db.delJail(self.jail)
+		jails = self.db.getJailNames()
+		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		jails = self.db.getJailNames(enabled=False)
+		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		jails = self.db.getJailNames(enabled=True)
+		self.assertTrue(len(jails) == 0)
+		# Add it again - should just enable it:
+		self.db.addJail(self.jail)
+		jails = self.db.getJailNames()
+		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		jails = self.db.getJailNames(enabled=True)
+		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		jails = self.db.getJailNames(enabled=False)
+		self.assertTrue(len(jails) == 0)
+
 	def testPurge(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 044e0346..df98e1ac 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -587,18 +587,18 @@ def get_monitor_failures_testcase(Filter_):
 		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			return Utils.wait_for(lambda: self.jail.isFilled(), delay)
+			return Utils.wait_for(self.jail.isFilled, delay)
 
 		def _sleep_4_poll(self):
 			# Since FilterPoll relies on time stamps and some
 			# actions might be happening too fast in the tests,
 			# sleep a bit to guarantee reliable time stamps
 			if isinstance(self.filter, FilterPoll):
-				Utils.wait_for(lambda: self.filter.isAlive(), 4*Utils.DEFAULT_SLEEP_TIME)
+				Utils.wait_for(self.filter.isAlive, 4*Utils.DEFAULT_SLEEP_TIME)
 
 		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
 			# shorter wait time for not modified status
-			return Utils.wait_for(lambda: self.jail.isEmpty(), delay)
+			return Utils.wait_for(self.jail.isEmpty, delay)
 
 		def assert_correct_last_attempt(self, failures, count=None):
 			self.assertTrue(self.isFilled(20)) # give Filter a chance to react
@@ -789,11 +789,11 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			return Utils.wait_for(lambda: self.jail.isFilled(), delay)
+			return Utils.wait_for(self.jail.isFilled, delay)
 
 		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
 			# shorter wait time for not modified status
-			return Utils.wait_for(lambda: self.jail.isEmpty(), delay)
+			return Utils.wait_for(self.jail.isEmpty, delay)
 
 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
@@ -1129,6 +1129,52 @@ class GetFailures(LogCaptureTestCase):
 
 class DNSUtilsTests(unittest.TestCase):
 
+	def testCache(self):
+		c = Utils.Cache(maxCount=5, maxTime=60)
+		# not available :
+		self.assertTrue(c.get('a') is None)
+		self.assertEqual(c.get('a', 'test'), 'test')
+		# exact 5 elements :
+		for i in xrange(5):
+			c.set(i, i)
+		for i in xrange(5):
+			self.assertEqual(c.get(i), i)
+
+	def testCacheMaxSize(self):
+		c = Utils.Cache(maxCount=5, maxTime=60)
+		# exact 5 elements :
+		for i in xrange(5):
+			c.set(i, i)
+		self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)])
+		self.assertFalse(-1 in [c.get(i, -1) for i in xrange(5)])
+		# add one - too many:
+		c.set(10, i)
+		# one element should be removed :
+		self.assertTrue(-1 in [c.get(i, -1) for i in xrange(5)])
+		# test max size (not expired):
+		for i in xrange(10):
+			c.set(i, 1)
+		self.assertEqual(len(c), 5)
+
+	def testCacheMaxTime(self):
+		# test max time (expired, timeout reached) :
+		c = Utils.Cache(maxCount=5, maxTime=0.0005)
+		for i in xrange(10):
+			c.set(i, 1)
+		st = time.time()
+		self.assertTrue(Utils.wait_for(lambda: time.time() >= st + 0.0005, 1))
+		# we have still 5 elements (or fewer if too slow test mashine):
+		self.assertTrue(len(c) <= 5)
+		# but all that are expiered also:
+		for i in xrange(10):
+			self.assertTrue(c.get(i) is None)
+		# here the whole cache should be empty:
+		self.assertEqual(len(c), 0)
+		
+
+
+class DNSUtilsNetworkTests(unittest.TestCase):
+
 	def setUp(self):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py
new file mode 100644
index 00000000..68a44bb5
--- /dev/null
+++ b/fail2ban/tests/tickettestcase.py
@@ -0,0 +1,176 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+
+__author__ = "Serg G. Brester (sebres)"
+__copyright__ = "Copyright (c) 2015 Serg G. Brester, 2015- Fail2Ban Contributors"
+__license__ = "GPL"
+
+from ..server.mytime import MyTime
+import unittest
+
+from ..server.ticket import Ticket, FailTicket, BanTicket
+
+
+class TicketTests(unittest.TestCase):
+
+  def testTicket(self):
+
+    tm = MyTime.time()
+    matches = ['first', 'second']
+    matches2 = ['first', 'second']
+    matches3 = ['first', 'second', 'third']
+
+    # Ticket
+    t = Ticket('193.168.0.128', tm, matches)
+    self.assertEqual(t.getIP(), '193.168.0.128')
+    self.assertEqual(t.getTime(), tm)
+    self.assertEqual(t.getMatches(), matches2)
+    t.setAttempt(2)
+    self.assertEqual(t.getAttempt(), 2)
+    t.setBanCount(10)
+    self.assertEqual(t.getBanCount(), 10)
+    # default ban time (from manager):
+    self.assertEqual(t.getBanTime(60*60), 60*60)
+    self.assertFalse(t.isTimedOut(tm + 60 + 1, 60*60))
+    self.assertTrue(t.isTimedOut(tm + 60*60 + 1, 60*60))
+    t.setBanTime(60)
+    self.assertEqual(t.getBanTime(60*60), 60)
+    self.assertEqual(t.getBanTime(), 60)
+    self.assertFalse(t.isTimedOut(tm))
+    self.assertTrue(t.isTimedOut(tm + 60 + 1))
+    # permanent :
+    t.setBanTime(-1)
+    self.assertFalse(t.isTimedOut(tm + 60 + 1))
+    t.setBanTime(60)
+
+    # BanTicket
+    tm = MyTime.time()
+    matches = ['first', 'second']
+    ft = FailTicket('193.168.0.128', tm, matches)
+    ft.setBanTime(60*60)
+    self.assertEqual(ft.getIP(), '193.168.0.128')
+    self.assertEqual(ft.getTime(), tm)
+    self.assertEqual(ft.getMatches(), matches2)
+    ft.setAttempt(2)
+    self.assertEqual(ft.getAttempt(), 2)
+    # retry is max of set retry and failures:
+    self.assertEqual(ft.getRetry(), 2)
+    ft.setRetry(1)
+    self.assertEqual(ft.getRetry(), 2)
+    ft.setRetry(3)
+    self.assertEqual(ft.getRetry(), 3)
+    ft.inc()
+    self.assertEqual(ft.getAttempt(), 3)
+    self.assertEqual(ft.getRetry(), 4)
+    self.assertEqual(ft.getMatches(), matches2)
+    # with 1 match, 1 failure and factor 10 (retry count) :
+    ft.inc(['third'], 1, 10)
+    self.assertEqual(ft.getAttempt(), 4)
+    self.assertEqual(ft.getRetry(), 14)
+    self.assertEqual(ft.getMatches(), matches3)
+    # last time (ignore if smaller as time):
+    self.assertEqual(ft.getLastTime(), tm)
+    ft.setLastTime(tm-60)
+    self.assertEqual(ft.getTime(), tm)
+    self.assertEqual(ft.getLastTime(), tm)
+    ft.setLastTime(tm+60)
+    self.assertEqual(ft.getTime(), tm+60)
+    self.assertEqual(ft.getLastTime(), tm+60)
+    ft.setData('country', 'DE')
+    self.assertEqual(ft.getData(),  
+      {'matches': ['first', 'second', 'third'], 'failures': 4, 'country': 'DE'})
+
+    # copy all from another ticket:
+    ft2 = FailTicket(ticket=ft)
+    self.assertEqual(ft, ft2)
+    self.assertEqual(ft.getData(), ft2.getData())
+    self.assertEqual(ft2.getAttempt(), 4)
+    self.assertEqual(ft2.getRetry(), 14)
+    self.assertEqual(ft2.getMatches(), matches3)
+    self.assertEqual(ft2.getTime(), ft.getTime())
+    self.assertEqual(ft2.getLastTime(), ft.getLastTime())
+    self.assertEqual(ft2.getBanTime(), ft.getBanTime())
+
+  def testTicketData(self):
+    t = BanTicket('193.168.0.128', None, ['first', 'second'])
+    # expand data (no overwrites, matches are available) :
+    t.setData('region', 'Hamburg', 'country', 'DE', 'city', 'Hamburg')
+    self.assertEqual(
+      t.getData(), 
+      {'matches': ['first', 'second'], 'failures':0, 'region': 'Hamburg', 'country': 'DE', 'city': 'Hamburg'})
+    # at once as dict (single argument, overwrites it completelly, no more matches/failures) :
+    t.setData({'region': None, 'country': 'FR', 'city': 'Paris'},)
+    self.assertEqual(
+      t.getData(), 
+      {'city': 'Paris', 'country': 'FR'})
+    # at once as dict (overwrites it completelly, no more matches/failures) :
+    t.setData({'region': 'Hamburg', 'country': 'DE', 'city': None})
+    self.assertEqual(
+      t.getData(), 
+      {'region': 'Hamburg', 'country': 'DE'})
+    self.assertEqual(
+      t.getData('region'), 
+      'Hamburg')
+    self.assertEqual(
+      t.getData('country'), 
+      'DE')
+    # again, named arguments:
+    t.setData(region='Bremen', city='Bremen')
+    self.assertEqual(t.getData(), 
+      {'region': 'Bremen', 'country': 'DE', 'city': 'Bremen'})
+    # again, but as args (key value pair):
+    t.setData('region', 'Brandenburg', 'city', 'Berlin')
+    self.assertEqual(
+      t.getData('region'), 
+      'Brandenburg')
+    self.assertEqual(
+      t.getData('city'), 
+      'Berlin')
+    self.assertEqual(
+      t.getData(), 
+      {'city':'Berlin', 'region': 'Brandenburg', 'country': 'DE'})
+    # interator filter :
+    self.assertEqual(
+      t.getData(('city', 'country')), 
+      {'city':'Berlin', 'country': 'DE'})
+    # callable filter :
+    self.assertEqual(
+      t.getData(lambda k: k.upper() == 'COUNTRY'),
+      {'country': 'DE'})
+    # remove one data entry:
+    t.setData('city', None)
+    self.assertEqual(
+      t.getData(), 
+      {'region': 'Brandenburg', 'country': 'DE'})
+    # default if not available:
+    self.assertEqual(
+      t.getData('city', 'Unknown'),
+      'Unknown')
+    # add continent :
+    t.setData('continent', 'Europe')
+    # again, but as argument list (overwrite new only, leave continent unchanged) :
+    t.setData(*['country', 'RU', 'region', 'Moscow'])
+    self.assertEqual(
+      t.getData(), 
+      {'continent': 'Europe', 'country': 'RU', 'region': 'Moscow'})
+    # clear:
+    t.setData({})
+    self.assertEqual(t.getData(), {})
+    self.assertEqual(t.getData('anything', 'default'), 'default')
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index a66325d8..017ad3f6 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -115,6 +115,7 @@ def gatherTests(regexps=None, opts=None):
 	# avoid circular imports
 	from . import banmanagertestcase
 	from . import clientreadertestcase
+	from . import tickettestcase
 	from . import failmanagertestcase
 	from . import filtertestcase
 	from . import servertestcase
@@ -150,6 +151,8 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(servertestcase.LoggingTests))
 	tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest))
 	tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions))
+	# Ticket, BanTicket, FailTicket
+	tests.addTest(unittest.makeSuite(tickettestcase.TicketTests))
 	# FailManager
 	tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
 	# BanManager
@@ -187,6 +190,7 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS))
 	tests.addTest(unittest.makeSuite(filtertestcase.GetFailures))
 	tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests))
+	tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsNetworkTests))
 	tests.addTest(unittest.makeSuite(filtertestcase.JailTests))
 
 	# DateDetector

From cf3cf27fa3c57c6a43448e23faa2b37c553395ee Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 17 Jul 2015 21:05:50 +0200
Subject: [PATCH 015/623] - filtering of test cases extended for method
 additionally to class (matches method only not whole class for regexp matched
 method name); - new options for "fail2ban-testcases" introduced: "-g" or
 "--no-gamin" and "-m" or "--memory-db", both are true also if "-f" or
 "--fast" specified,   for example: `fail2ban-testcases -ngm` will runs faster
 (because no network, no gamin, memory database)   but will use the same
 default sleep intervals as in production (in comparison to -nf); - seekToTime
 rewritten, accuracy increased by seekToTime, extended for all FileFilter
 (PyInotify, Gamin also), test cases extended etc. - common performance
 optimized and code reviewed;

---
 bin/fail2ban-testcases             |   6 +
 fail2ban/server/filter.py          | 119 +++++++++------
 fail2ban/server/filterpoll.py      |   7 +-
 fail2ban/server/ticket.py          |   6 +
 fail2ban/tests/databasetestcase.py |  26 ++--
 fail2ban/tests/filtertestcase.py   | 228 +++++++++++++++++++++--------
 fail2ban/tests/servertestcase.py   |   7 +-
 fail2ban/tests/utils.py            |  45 +++---
 8 files changed, 301 insertions(+), 143 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 28af8265..22a44a8b 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -58,6 +58,12 @@ def get_opt_parser():
 		Option('-n', "--no-network", action="store_true",
 			   dest="no_network",
 			   help="Do not run tests that require the network"),
+		Option('-g', "--no-gamin", action="store_true",
+			   dest="no_gamin",
+			   help="Do not run tests that require the gamin"),
+		Option('-m', "--memory-db", action="store_true",
+			   dest="memory_db",
+			   help="Run database tests using memory instead of file"),
 		Option('-f', "--fast", action="store_true",
 			   dest="fast",
 			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e5cf648c..d1ac3680 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -560,6 +560,7 @@ class FileFilter(Filter):
 		Filter.__init__(self, jail, **kwargs)
 		## The log file path.
 		self.__logs = dict()
+		self.__autoSeek = dict()
 		self.setLogEncoding("auto")
 
 	##
@@ -567,7 +568,7 @@ class FileFilter(Filter):
 	#
 	# @param path log file path
 
-	def addLogPath(self, path, tail=False):
+	def addLogPath(self, path, tail=False, autoSeek=True):
 		if path in self.__logs:
 			logSys.error(path + " already exists")
 		else:
@@ -579,6 +580,11 @@ class FileFilter(Filter):
 					log.setPos(lastpos)
 			self.__logs[path] = log
 			logSys.info("Added logfile = %s (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
+			if autoSeek:
+				# if default, seek to "current time" - "find time":
+				if isinstance(autoSeek, bool):
+					autoSeek = MyTime.time() - self.getFindTime()
+				self.__autoSeek[path] = autoSeek
 			self._addLogPath(path)			# backend specific
 
 	def _addLogPath(self, path):
@@ -673,7 +679,7 @@ class FileFilter(Filter):
 	# MyTime.time()-self.findTime. When a failure is detected, a FailTicket
 	# is created and is added to the FailManager.
 
-	def getFailures(self, filename, startTime=None):
+	def getFailures(self, filename):
 		log = self.getLog(filename)
 		if log is None:
 			logSys.error("Unable to get failures in " + filename)
@@ -695,13 +701,17 @@ class FileFilter(Filter):
 			logSys.exception(e)
 			return False
 
-		# prevent completely read of big files first time (after start of service), initial seek to start time using half-interval search algorithm:
-		if log.getPos() == 0 and startTime is not None:
+		# seek to find time for first usage only (prevent performance decline with polling of big files)
+		if self.__autoSeek.get(filename):
+			startTime = self.__autoSeek[filename]
+			del self.__autoSeek[filename]
+			# prevent completely read of big files first time (after start of service), 
+			# initial seek to start time using half-interval search algorithm:
 			try:
-				# startTime = MyTime.time() - self.getFindTime()
 				self.seekToTime(log, startTime)
 			except Exception, e: # pragma: no cover
 				logSys.error("Error during seek to start time in \"%s\"", filename)
+				raise
 				logSys.exception(e)
 				return False
 
@@ -726,71 +736,88 @@ class FileFilter(Filter):
 	# Seeks to line with date (search using half-interval search algorithm), to start polling from it
 	#
 
-	def seekToTime(self, container, date):
+	def seekToTime(self, container, date, accuracy=3):
 		fs = container.getFileSize()
 		if logSys.getEffectiveLevel() <= logging.DEBUG:
 			logSys.debug("Seek to find time %s (%s), file size %s", date, 
 				datetime.datetime.fromtimestamp(date).strftime("%Y-%m-%d %H:%M:%S"), fs)
-		date -= 0.009
-		minp = 0
+		minp = container.getPos()
 		maxp = fs
-		lastpos = 0
-		lastFew = 0
-		lastTime = None
+		tryPos = minp
+		lastPos = -1
+		foundPos = 0
+		foundTime = None
 		cntr = 0
 		unixTime = None
-		lasti = 0
-		movecntr = 1
+		movecntr = accuracy
 		while maxp > minp:
-			i = int(minp + (maxp - minp) / 2)
-			pos = container.seek(i)
+			if tryPos is None:
+				pos = int(minp + (maxp - minp) / 2)
+			else:
+				pos, tryPos = tryPos, None
+			# because container seek will go to start of next line (minus CRLF):
+			pos = max(0, pos-2)
+			seekpos = pos = container.seek(pos)
 			cntr += 1
 			# within next 5 lines try to find any legal datetime:
 			lncntr = 5;
 			dateTimeMatch = None
-			llen = 0
-			if lastpos == pos:
-				i = pos
+			nextp = None
 			while True:
 				line = container.readline()
 				if not line:
 					break
-				llen += len(line)
-				l = line.rstrip('\r\n')
-				(timeMatch, template) = self.dateDetector.matchTime(l)
+				(timeMatch, template) = self.dateDetector.matchTime(line)
 				if timeMatch:
-					dateTimeMatch = self.dateDetector.getTime2(l[timeMatch.start():timeMatch.end()], (timeMatch, template))
+					dateTimeMatch = self.dateDetector.getTime2(line[timeMatch.start():timeMatch.end()], (timeMatch, template))
+				else:
+					nextp = container.tell()
+					if nextp > maxp:
+						pos = seekpos
+						break
+					pos = nextp
 				if not dateTimeMatch and lncntr:
 					lncntr -= 1
 					continue
 				break
+		 	# not found at this step - stop searching
+			if dateTimeMatch:
+				unixTime = dateTimeMatch[0]
+				if unixTime >= date:
+					if foundTime is None or unixTime <= foundTime:
+						foundPos = pos
+						foundTime = unixTime
+					if pos == maxp:
+						pos = seekpos
+					if pos < maxp:
+						maxp = pos
+				else:
+					if foundTime is None or unixTime >= foundTime:
+						foundPos = pos
+						foundTime = unixTime
+					if nextp is None:
+						nextp = container.tell()
+					pos = nextp
+					if pos > minp:
+						minp = pos
 			# if we can't move (position not changed)
-			if i + llen == lasti:
+			if pos == lastPos:
 				movecntr -= 1
 				if movecntr <= 0:
 		 			break
-			lasti = i + llen;
-		 	# not found at this step - stop searching
-			if not dateTimeMatch:
+				# we have found large area without any date mached 
+				# or end of search - try min position (because can be end of previous line):
+				if minp != lastPos:
+					lastPos = tryPos = minp
+					continue
 				break
-			unixTime = dateTimeMatch[0]
-			if unixTime >= date:
-				maxp = i
-			else:
-				minp = i + llen
-				lastFew = pos;
-				lastTime = unixTime
-			lastpos = pos
-		# if found position have a time greater as given - use smallest time we have found
-		if unixTime is None or unixTime > date:
-			unixTime = lastTime
-			lastpos = container.seek(lastFew, False)
-		else:
-			lastpos = container.seek(lastpos, False)
-		container.setPos(lastpos)
+			lastPos = pos
+		# always use smallest pos, that could be found:
+		foundPos = container.seek(minp, False)
+		container.setPos(foundPos)
 		if logSys.getEffectiveLevel() <= logging.DEBUG:
-			logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastpos, fs, unixTime, 
-				(datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") if unixTime is not None else ''), cntr)
+			logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastPos, fs, foundTime, 
+				(datetime.datetime.fromtimestamp(foundTime).strftime("%Y-%m-%d %H:%M:%S") if foundTime is not None else ''), cntr)
 		
 	def status(self, flavor="basic"):
 		"""Status of Filter plus files being monitored.
@@ -898,11 +925,15 @@ class FileContainer:
 		# seek to given position
 		h.seek(offs, 0)
 		# goto end of next line
-		if endLine:
+		if offs and endLine:
 			h.readline()
 		# get current real position
 		return h.tell()
 
+	def tell(self):
+		# get current real position
+		return self.__handler.tell()
+
 	@staticmethod
 	def decode_line(filename, enc, line):
 		try:
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 1afda7f2..d7ff9075 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -58,7 +58,6 @@ class FilterPoll(FileFilter):
 		## The time of the last modification of the file.
 		self.__prevStats = dict()
 		self.__file404Cnt = dict()
-		self.__initial = dict()
 		logSys.debug("Created FilterPoll")
 
 	##
@@ -109,11 +108,7 @@ class FilterPoll(FileFilter):
 			modlst = []
 			Utils.wait_for(lambda: self.getModified(modlst), self.sleeptime)
 			for filename in modlst:
-				# set start time as now - find time for first usage only (prevent performance bug with polling of big files)
-				self.getFailures(filename, 
-					(MyTime.time() - self.getFindTime()) if not self.__initial.get(filename) else None
-				)
-				self.__initial[filename] = True
+				self.getFailures(filename)
 				self.__modified = True
 
 			if self.__modified:
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index e856b66d..b3eea052 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -176,6 +176,9 @@ class FailTicket(Ticket):
 			self.__retry = self._data['failures'];
 
 	def setRetry(self, value):
+		""" Set artificial retry count, normally equal failures / attempt,
+		used in incremental features (BanTimeIncr) to increase retry count for bad IPs
+		"""
 		self.__retry = value
 		if not self._data['failures']:
 			self._data['failures'] = 1
@@ -184,6 +187,9 @@ class FailTicket(Ticket):
 			self._data['matches'] = []
 
 	def getRetry(self):
+		""" Returns failures / attempt count or
+		artificial retry count increased for bad IPs
+		"""
 		return max(self.__retry, self._data['failures'])
 
 	def inc(self, matches=None, attempt=1, count=1):
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 3f0e4c10..6b85af56 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -36,18 +36,20 @@ from ..server.actions import Actions
 from .dummyjail import DummyJail
 try:
 	from ..server.database import Fail2BanDb as Fail2BanDb
-	# because of tests performance use memory instead of file:
-	def TestFail2BanDb(filename):
-		if unittest.F2B.fast:
-			return Fail2BanDb(':memory:')
-		return Fail2BanDb(filename)
-except ImportError:
+except ImportError: # pragma: no cover
 	Fail2BanDb = None
 from .utils import LogCaptureTestCase
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 
+# because of tests performance use memory instead of file:
+def getFail2BanDb(filename):
+	if unittest.F2B.memory_db: # pragma: no cover
+		return Fail2BanDb(':memory:')
+	return Fail2BanDb(filename)
+
+
 class DatabaseTest(LogCaptureTestCase):
 
 	def setUp(self):
@@ -59,8 +61,10 @@ class DatabaseTest(LogCaptureTestCase):
 				"available.")
 		elif Fail2BanDb is None:
 			return
-		_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
-		self.db = TestFail2BanDb(self.dbFilename)
+		self.dbFilename = None
+		if not unittest.F2B.memory_db:
+			_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
+		self.db = getFail2BanDb(self.dbFilename)
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -68,7 +72,8 @@ class DatabaseTest(LogCaptureTestCase):
 		if Fail2BanDb is None: # pragma: no cover
 			return
 		# Cleanup
-		os.remove(self.dbFilename)
+		if self.dbFilename is not None:
+			os.remove(self.dbFilename)
 
 	def testGetFilename(self):
 		if Fail2BanDb is None or self.db.filename == ':memory:': # pragma: no cover
@@ -106,6 +111,9 @@ class DatabaseTest(LogCaptureTestCase):
 	def testUpdateDb(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
+		self.db = None
+		if self.dbFilename is None: # pragma: no cover
+			_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
 		shutil.copyfile(
 			os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename)
 		self.db = Fail2BanDb(self.dbFilename)
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index df98e1ac..11642d2c 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -81,6 +81,16 @@ def _killfile(f, name):
 		_killfile(None, name + '.bak')
 
 
+def _maxWaitTime(wtime):
+	if unittest.F2B.fast:
+		wtime /= 10
+	return wtime
+
+
+def _tm(time):
+	return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S")
+
+
 def _assert_equal_entries(utest, found, output, count=None):
 	"""Little helper to unify comparisons with the target entries
 
@@ -122,9 +132,15 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None):
 	Test filter to contain target ticket
 	"""
 	if isinstance(filter_, DummyJail):
+		# get fail ticket from jail
 		found = _ticket_tuple(filter_.getFailTicket())
 	else:
 		# when we are testing without jails
+		# wait for failures (up to max time)
+		Utils.wait_for(
+			lambda: filter_.failManager.getFailTotal() >= (count if count else output[1]),
+			_maxWaitTime(10))
+		# get fail ticket from filter
 		found = _ticket_tuple(filter_.failManager.toBan())
 
 	_assert_equal_entries(utest, found, output, count)
@@ -352,59 +368,132 @@ class LogFileFilterPoll(unittest.TestCase):
 		self.assertTrue(self.filter.isModified(LogFileFilterPoll.FILENAME))
 		self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME))
 
-	def testSeekToTime(self):
+	def testSeekToTimeSmallFile(self):
 		fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log')
-		tm = lambda time: datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S")
 		time = 1417512352
 		f = open(fname, 'w')
-		fc = FileContainer(fname, self.filter.getLogEncoding())
-		fc.open()
-		fc.setPos(0); self.filter.seekToTime(fc, time)
+		fc = None
 		try:
+			fc = FileContainer(fname, self.filter.getLogEncoding())
+			fc.open()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
 			f.flush()
 			# empty :
 			fc.setPos(0); self.filter.seekToTime(fc, time)
 			self.assertEqual(fc.getPos(), 0)
 			# one entry with exact time:
-			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time))
+			f.write("%s [sshd] error: PAM: failure len 1\n" % _tm(time))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
-			# one entry with smaller time:
+
+			# rewrite :
 			f.seek(0)
-			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time - 10))
+			f.truncate()
+			fc.close()
+			fc = FileContainer(fname, self.filter.getLogEncoding())
+			fc.open()
+			# no time - nothing should be found :
+			for i in xrange(10):
+				f.write("[sshd] error: PAM: failure len 1\n")
+				f.flush()
+				fc.setPos(0); self.filter.seekToTime(fc, time)
+
+			# rewrite
+			f.seek(0)
+			f.truncate()
+			fc.close()
+			fc = FileContainer(fname, self.filter.getLogEncoding())
+			fc.open()
+			# one entry with smaller time:
+			f.write("%s [sshd] error: PAM: failure len 2\n" % _tm(time - 10))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
-			self.assertEqual(fc.getPos(), 0)
-			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time - 9))
-			f.flush()
-			fc.setPos(0); self.filter.seekToTime(fc, time)
-			self.assertEqual(fc.getPos(), 0)
-			# add exact time between:
-			f.write("%s [sshd] error: PAM: failure\n" % tm(time - 1))
+			self.assertEqual(fc.getPos(), 53)
+			# two entries with smaller time:
+			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % _tm(time - 9))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
 			self.assertEqual(fc.getPos(), 110)
+			# check move after end (all of time smaller):
+			f.write("%s [sshd] error: PAM: failure\n" % _tm(time - 1))
+			f.flush()
+			self.assertEqual(fc.getFileSize(), 157)
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 157)
+
 			# stil one exact line:
-			f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time))
-			f.write("%s [sshd] error: PAM: failure len 1\n" % tm(time))
+			f.write("%s [sshd] error: PAM: Authentication failure\n" % _tm(time))
+			f.write("%s [sshd] error: PAM: failure len 1\n" % _tm(time))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
-			self.assertEqual(fc.getPos(), 110)
+			self.assertEqual(fc.getPos(), 157)
+
 			# add something hereafter:
-			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 2))
-			f.write("%s [sshd] error: PAM: Authentication failure\n" % tm(time + 3))
+			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % _tm(time + 2))
+			f.write("%s [sshd] error: PAM: Authentication failure\n" % _tm(time + 3))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
-			self.assertEqual(fc.getPos(), 110)
+			self.assertEqual(fc.getPos(), 157)
 			# add something hereafter:
-			f.write("%s [sshd] error: PAM: failure\n" % tm(time + 9))
-			f.write("%s [sshd] error: PAM: failure len 3 2 1\n" % tm(time + 9))
+			f.write("%s [sshd] error: PAM: failure\n" % _tm(time + 9))
+			f.write("%s [sshd] error: PAM: failure len 4 3 2\n" % _tm(time + 9))
 			f.flush()
 			fc.setPos(0); self.filter.seekToTime(fc, time)
-			self.assertEqual(fc.getPos(), 110)
+			self.assertEqual(fc.getPos(), 157)
+			# start search from current pos :
+			fc.setPos(157); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 157)
+			# start search from current pos :
+			fc.setPos(110); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 157)
 
 		finally:
-			fc.close()
+			if fc:
+				fc.close()
+			_killfile(f, fname)
+
+	def testSeekToTimeLargeFile(self):
+		fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log')
+		time = 1417512352
+		f = open(fname, 'w')
+		fc = None
+		count = 1000 if unittest.F2B.fast else 10000
+		try:
+			fc = FileContainer(fname, self.filter.getLogEncoding())
+			fc.open()
+			f.seek(0)
+			# variable length of file (ca 45K or 450K before and hereafter):
+			# write lines with smaller as search time:
+			t = time - count - 1
+			for i in xrange(count):
+				f.write("%s [sshd] error: PAM: failure\n" % _tm(t))
+				t += 1
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 47*count)
+			# write lines with exact search time:
+			for i in xrange(10):
+				f.write("%s [sshd] error: PAM: failure\n" % _tm(time))
+			f.flush()
+			fc.setPos(0); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 47*count)
+			fc.setPos(4*count); self.filter.seekToTime(fc, time)
+			self.assertEqual(fc.getPos(), 47*count)
+			# write lines with greater as search time:
+			t = time+1
+			for i in xrange(count//500):
+				for j in xrange(500):
+					f.write("%s [sshd] error: PAM: failure\n" % _tm(t))
+					t += 1
+				f.flush()
+				fc.setPos(0); self.filter.seekToTime(fc, time)
+				self.assertEqual(fc.getPos(), 47*count)
+				fc.setPos(53); self.filter.seekToTime(fc, time)
+				self.assertEqual(fc.getPos(), 47*count)
+		
+		finally:
+			if fc:
+				fc.close()
 			_killfile(f, fname)
 
 class LogFileMonitor(LogCaptureTestCase):
@@ -418,7 +507,7 @@ class LogFileMonitor(LogCaptureTestCase):
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
 		self.file = open(self.name, 'a')
 		self.filter = FilterPoll(DummyJail())
-		self.filter.addLogPath(self.name)
+		self.filter.addLogPath(self.name, autoSeek=False)
 		self.filter.active = True
 		self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
 
@@ -431,11 +520,12 @@ class LogFileMonitor(LogCaptureTestCase):
 	def isModified(self, delay=2.):
 		"""Wait up to `delay` sec to assure that it was modified or not
 		"""
-		return Utils.wait_for(lambda: self.filter.isModified(self.name), delay)
+		return Utils.wait_for(lambda: self.filter.isModified(self.name), _maxWaitTime(delay))
 
-	def notModified(self):
-		# shorter wait time for not modified status
-		return not self.isModified(4*Utils.DEFAULT_SLEEP_TIME)
+	def notModified(self, delay=2.):
+		"""Wait up to `delay` sec as long as it was not modified
+		"""
+		return Utils.wait_for(lambda: not self.filter.isModified(self.name), _maxWaitTime(delay))
 
 	def testUnaccessibleLogFile(self):
 		os.chmod(self.name, 0)
@@ -478,7 +568,7 @@ class LogFileMonitor(LogCaptureTestCase):
 		os.rename(self.name, self.name + '.old')
 		# we are not signaling as modified whenever
 		# it gets away
-		self.assertTrue(self.notModified())
+		self.assertTrue(self.notModified(1))
 		f = open(self.name, 'a')
 		self.assertTrue(self.isModified())
 		self.assertTrue(self.notModified())
@@ -562,7 +652,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = open(self.name, 'a')
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
-			self.filter.addLogPath(self.name)
+			self.filter.addLogPath(self.name, autoSeek=False)
 			self.filter.active = True
 			self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
 			self.filter.start()
@@ -587,21 +677,21 @@ def get_monitor_failures_testcase(Filter_):
 		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			return Utils.wait_for(self.jail.isFilled, delay)
+			return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
 
 		def _sleep_4_poll(self):
 			# Since FilterPoll relies on time stamps and some
 			# actions might be happening too fast in the tests,
 			# sleep a bit to guarantee reliable time stamps
 			if isinstance(self.filter, FilterPoll):
-				Utils.wait_for(self.filter.isAlive, 4*Utils.DEFAULT_SLEEP_TIME)
+				Utils.wait_for(self.filter.isAlive, _maxWaitTime(5))
 
-		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
+		def isEmpty(self, delay=_maxWaitTime(5)):
 			# shorter wait time for not modified status
-			return Utils.wait_for(self.jail.isEmpty, delay)
+			return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
 
 		def assert_correct_last_attempt(self, failures, count=None):
-			self.assertTrue(self.isFilled(20)) # give Filter a chance to react
+			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
 			_assert_correct_last_attempt(self, self.jail, failures, count=count)
 
 		def test_grow_file(self):
@@ -616,7 +706,7 @@ def get_monitor_failures_testcase(Filter_):
 			# since it should have not been enough
 
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5)
-			self.assertTrue(self.isFilled(6))
+			self.assertTrue(self.isFilled(10))
 			# so we sleep for up to 2 sec for it not to become empty,
 			# and meanwhile pass to other thread(s) and filter should
 			# have gathered new failures and passed them into the
@@ -653,11 +743,11 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  n=14, mode='w')
 			# Poll might need more time
-			self.assertTrue(self.isEmpty(min(4, 100 * Utils.DEFAULT_SLEEP_TIME)),
+			self.assertTrue(self.isEmpty(_maxWaitTime(5)),
 							"Queue must be empty but it is not: %s."
 							% (', '.join([str(x) for x in self.jail.queue])))
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
-			Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, 50 * Utils.DEFAULT_SLEEP_TIME)
+			Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, _maxWaitTime(10))
 			self.assertEqual(self.filter.failManager.getFailTotal(), 2)
 
 			# move aside, but leaving the handle still open...
@@ -682,7 +772,7 @@ def get_monitor_failures_testcase(Filter_):
 
 			if interim_kill:
 				_killfile(None, self.name)
-				time.sleep(Utils.DEFAULT_SLEEP_TIME)				  # let them know
+				time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)				  # let them know
 
 			# now create a new one to override old one
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new',
@@ -729,10 +819,10 @@ def get_monitor_failures_testcase(Filter_):
 
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
 			# so we should get no more failures detected
-			self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME))
+			self.assertTrue(self.isEmpty(_maxWaitTime(10)))
 
-			# but then if we add it back again
-			self.filter.addLogPath(self.name)
+			# but then if we add it back again (no seek to time in FileFilter's, because in file used the same time)
+			self.filter.addLogPath(self.name, autoSeek=False)
 			# Tricky catch here is that it should get them from the
 			# tail written before, so let's not copy anything yet
 			#_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100)
@@ -789,14 +879,14 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
-			return Utils.wait_for(self.jail.isFilled, delay)
+			return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
 
-		def isEmpty(self, delay=4*Utils.DEFAULT_SLEEP_TIME):
+		def isEmpty(self, delay=_maxWaitTime(5)):
 			# shorter wait time for not modified status
-			return Utils.wait_for(self.jail.isEmpty, delay)
+			return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
 
 		def assert_correct_ban(self, test_ip, test_attempts):
-			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
+			self.assertTrue(self.isFilled(_maxWaitTime(10))) # give Filter a chance to react
 			ticket = self.jail.getFailTicket()
 
 			attempts = ticket.getAttempt()
@@ -819,7 +909,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, skip=2, n=3)
-			self.assertTrue(self.isFilled(6))
+			self.assertTrue(self.isFilled(10))
 			# so we sleep for up to 6 sec for it not to become empty,
 			# and meanwhile pass to other thread(s) and filter should
 			# have gathered new failures and passed them into the
@@ -852,7 +942,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, n=5, skip=5)
 			# so we should get no more failures detected
-			self.assertTrue(self.isEmpty(200 * Utils.DEFAULT_SLEEP_TIME))
+			self.assertTrue(self.isEmpty(_maxWaitTime(10)))
 
 			# but then if we add it back again
 			self.filter.addJournalMatch([
@@ -863,7 +953,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, n=6, skip=10)
 			# we should detect the failures
-			self.assertTrue(self.isFilled(6))
+			self.assertTrue(self.isFilled(10))
 
 	return MonitorJournalFailures
 
@@ -931,7 +1021,7 @@ class GetFailures(LogCaptureTestCase):
 		filename = filename or GetFailures.FILENAME_01
 		failures = failures or GetFailures.FAILURES_01
 
-		self.filter.addLogPath(filename)
+		self.filter.addLogPath(filename, autoSeek=0)
 		self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>$")
 		self.filter.getFailures(filename)
 		_assert_correct_last_attempt(self, self.filter,  failures)
@@ -955,7 +1045,7 @@ class GetFailures(LogCaptureTestCase):
 				  [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2'
 				   % m for m in 53, 54, 57, 58])
 
-		self.filter.addLogPath(GetFailures.FILENAME_02)
+		self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0)
 		self.filter.addFailRegex("Failed .* from <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_02)
 		_assert_correct_last_attempt(self, self.filter, output)
@@ -963,25 +1053,35 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailures03(self):
 		output = ('203.162.223.135', 7, 1124013544.0)
 
-		self.filter.addLogPath(GetFailures.FILENAME_03)
+		self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0)
 		self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
 		self.filter.getFailures(GetFailures.FILENAME_03)
 		_assert_correct_last_attempt(self, self.filter, output)
 
-	def testGetFailures03_seek(self):
+	def testGetFailures03_Seek1(self):
 		# same test as above but with seek to 'Aug 14 11:55:04' - so other output ...
 		output = ('203.162.223.135', 5, 1124013544.0)
 
-		self.filter.addLogPath(GetFailures.FILENAME_03)
+		self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60)
 		self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
-		self.filter.getFailures(GetFailures.FILENAME_03, output[2] - 4*60 + 1)
+		self.filter.getFailures(GetFailures.FILENAME_03)
+		_assert_correct_last_attempt(self, self.filter, output)
+
+	def testGetFailures03_Seek2(self):
+		# same test as above but with seek to 'Aug 14 11:59:04' - so other output ...
+		output = ('203.162.223.135', 1, 1124013544.0)
+		self.filter.setMaxRetry(1)
+
+		self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2])
+		self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
+		self.filter.getFailures(GetFailures.FILENAME_03)
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures04(self):
 		output = [('212.41.96.186', 4, 1124013600.0),
 				  ('212.41.96.185', 4, 1124017198.0)]
 
-		self.filter.addLogPath(GetFailures.FILENAME_04)
+		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_04)
 
@@ -1048,7 +1148,7 @@ class GetFailures(LogCaptureTestCase):
 			filter_.active = True
 			filter_.failManager.setMaxRetry(1)	# we might have just few failures
 
-			filter_.addLogPath(GetFailures.FILENAME_USEDNS)
+			filter_.addLogPath(GetFailures.FILENAME_USEDNS, autoSeek=False)
 			filter_.addFailRegex("Failed .* from <HOST>")
 			filter_.getFailures(GetFailures.FILENAME_USEDNS)
 			_assert_correct_last_attempt(self, filter_, output)
@@ -1056,14 +1156,14 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresMultiRegex(self):
 		output = ('141.3.81.106', 8, 1124013541.0)
 
-		self.filter.addLogPath(GetFailures.FILENAME_02)
+		self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False)
 		self.filter.addFailRegex("Failed .* from <HOST>")
 		self.filter.addFailRegex("Accepted .* from <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_02)
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailuresIgnoreRegex(self):
-		self.filter.addLogPath(GetFailures.FILENAME_02)
+		self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False)
 		self.filter.addFailRegex("Failed .* from <HOST>")
 		self.filter.addFailRegex("Accepted .* from <HOST>")
 		self.filter.addIgnoreRegex("for roehl")
@@ -1075,7 +1175,7 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresMultiLine(self):
 		output = [("192.0.43.10", 2, 1124013599.0),
 			("192.0.43.11", 1, 1124013598.0)]
-		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
@@ -1093,7 +1193,7 @@ class GetFailures(LogCaptureTestCase):
 
 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
-		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addIgnoreRegex("rsync error: Received SIGINT")
 		self.filter.setMaxLines(100)
@@ -1109,7 +1209,7 @@ class GetFailures(LogCaptureTestCase):
 		output = [("192.0.43.10", 2, 1124013599.0),
 			("192.0.43.11", 1, 1124013598.0),
 			("192.0.43.15", 1, 1124013598.0)]
-		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")
 		self.filter.setMaxLines(100)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index eedc8c94..d26d7562 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -76,12 +76,15 @@ class TransmitterBase(unittest.TestCase):
 		self.server.quit()
 
 	def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False):
+		"""Process set/get commands and compare both return values 
+		with outValue if it was given otherwise with inValue"""
 		setCmd = ["set", cmd, inValue]
 		getCmd = ["get", cmd]
 		if jail is not None:
 			setCmd.insert(1, jail)
 			getCmd.insert(1, jail)
 
+		# if outValue was not given (now None is allowed return/compare value also)
 		if outValue == (None,):
 			outValue = inValue
 
@@ -173,7 +176,7 @@ class Transmitter(TransmitterBase):
 			self.assertEqual(self.transm.proceed(["sleep", "0.0001"]), (0, None))
 
 	def testDatabase(self):
-		if not unittest.F2B.fast:
+		if not unittest.F2B.memory_db:
 			tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_")
 		else: # pragma: no cover
 			tmpFilename = ':memory:'
@@ -208,7 +211,7 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(self.transm.proceed(
 			["set", "dbfile", "None"]),
 			(0, None))
-		if not unittest.F2B.fast:
+		if not unittest.F2B.memory_db:
 			os.close(tmp)
 			os.unlink(tmpFilename)
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 017ad3f6..338ccec1 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko"
 __license__ = "GPL"
 
 import logging
+import optparse
 import os
 import re
 import sys
@@ -35,6 +36,7 @@ from ..server.filter import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 
+
 logSys = getLogger(__name__)
 
 CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None)
@@ -47,10 +49,14 @@ if not CONFIG_DIR:
 		CONFIG_DIR = '/etc/fail2ban'
 
 
-class F2B():
-	def __init__(self, fast=False, no_network=False):
-		self.fast=fast
-		self.no_network=no_network
+class F2B(optparse.Values):
+	def __init__(self, opts={}):
+		self.__dict__ = opts.__dict__ if opts else {
+			'fast': False, 'memory_db':False, 'no_gamin': False, 'no_network': False, 
+		}
+		if self.fast:
+			self.memory_db = True
+			self.no_gamin = True
 	def SkipIfFast(self):
 		pass
 	def SkipIfNoNetwork(self):
@@ -58,13 +64,11 @@ class F2B():
 
 
 def initTests(opts):
-	if opts: # pragma: no cover
-		unittest.F2B = F2B(opts.fast, opts.no_network)
-	else:
-		unittest.F2B = F2B()
+	unittest.F2B = F2B(opts)
 	# --fast :
 	if unittest.F2B.fast: # pragma: no cover
-		# prevent long sleeping during test cases...
+		# racy decrease default sleep intervals to test it faster 
+		# (prevent long sleeping during test cases ... less time goes to sleep):
 		Utils.DEFAULT_SLEEP_TIME = 0.0025
 		Utils.DEFAULT_SLEEP_INTERVAL = 0.0005
 		def F2B_SkipIfFast():
@@ -74,7 +78,7 @@ def initTests(opts):
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):
-			if (v > Utils.DEFAULT_SLEEP_TIME):
+			if (v > Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
 				raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v)
 			_org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME))
 		time.sleep = _new_sleep
@@ -103,7 +107,7 @@ def setUpMyTime():
 
 def tearDownMyTime():
 	os.environ.pop('TZ')
-	if old_TZ:
+	if old_TZ: # pragma: no cover
 		os.environ['TZ'] = old_TZ
 	time.tzset()
 	MyTime.myTime = None
@@ -135,11 +139,15 @@ def gatherTests(regexps=None, opts=None):
 			_regexps = [re.compile(r) for r in regexps]
 
 			def addTest(self, suite):
-				suite_str = str(suite)
-				for r in self._regexps:
-					if r.search(suite_str):
-						super(FilteredTestSuite, self).addTest(suite)
-						return
+				matched = []
+				for test in suite:
+					s = str(test)
+					for r in self._regexps:
+						if r.search(s):
+							matched.append(test)
+							break
+				for test in matched:
+					super(FilteredTestSuite, self).addTest(test)
 
 		tests = FilteredTestSuite()
 
@@ -160,7 +168,7 @@ def gatherTests(regexps=None, opts=None):
 	try:
 		import dns
 		tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo))
-	except ImportError:
+	except ImportError: # pragma: no cover
 		pass
 	# ClientReaders
 	tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
@@ -225,7 +233,8 @@ def gatherTests(regexps=None, opts=None):
 	try:
 		# because gamin can be very slow on some platforms (and can produce many failures 
 		# with fast sleep interval) - skip it by fast run:
-		unittest.F2B.SkipIfFast()
+		if unittest.F2B.fast or unittest.F2B.no_gamin: # pragma: no cover
+			raise Exception('Skip, fast: %s, no_gamin: %s' % (unittest.F2B.fast, unittest.F2B.no_gamin))
 		from ..server.filtergamin import FilterGamin
 		filters.append(FilterGamin)
 	except Exception, e: # pragma: no cover

From 770c219ab69411029910b4ee2d9cb8bc7b25bccc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 22 Jul 2015 17:32:37 +0200
Subject: [PATCH 016/623] SetupTest should run at version of python currently
 installed (resp. active one), skip if started with another version.

---
 fail2ban/tests/misctestcase.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index a2310ec5..c4dd716d 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -66,6 +66,17 @@ class HelpersTest(unittest.TestCase):
 		self.assertEqual(splitcommaspace(' 1, 2 , '), ['1', '2'])
 
 
+def _getSysPythonVersion():
+	import subprocess, locale
+	sysVerCmd = "python -c 'import sys; print(tuple(sys.version_info))'"
+	if sys.version_info >= (2,7):
+		sysVer = subprocess.check_output(sysVerCmd, shell=True)
+	else:
+		sysVer = subprocess.Popen(sysVerCmd, shell=True, stdout=subprocess.PIPE).stdout.read()
+	if sys.version_info >= (3,):
+		sysVer = sysVer.decode(locale.getpreferredencoding(), 'replace')
+	return str(sysVer).rstrip()
+
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
@@ -75,6 +86,12 @@ class SetupTest(unittest.TestCase):
 			raise unittest.SkipTest(
 				"Seems to be running not out of source distribution"
 				" -- cannot locate setup.py")
+		# compare current version of python installed resp. active one:
+		sysVer = _getSysPythonVersion()
+		if sysVer != str(tuple(sys.version_info)):
+			raise unittest.SkipTest(
+				"Seems to be running with python distribution %s"
+				" -- install can be tested only with system distribution %s" % (str(tuple(sys.version_info)), sysVer))
 
 	def testSetupInstallRoot(self):
 		if not self.setup:

From 72f29e90619dbc02bc3192c9f44ddf38195a2393 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Jul 2015 20:23:07 +0200
Subject: [PATCH 017/623] asyncserver (asyncore) code fixed and test cases
 repaired (always delete temp files, wait for end of thread/server, etc)
 definitely closes gh-161, also other usage of asyncore event loop (in
 test_smtp.py) repair cache in ipToName (can returns None), precaching of
 invalid IPs (according to RFC 5737) to stop endless wait for resolving it in
 test cases.

---
 fail2ban/server/asyncserver.py       | 67 ++++++++++++++++++++--------
 fail2ban/server/filter.py            |  4 +-
 fail2ban/tests/action_d/test_smtp.py | 10 ++---
 fail2ban/tests/servertestcase.py     |  6 +++
 fail2ban/tests/sockettestcase.py     | 32 +++++++++++--
 fail2ban/tests/utils.py              | 13 ++++++
 6 files changed, 103 insertions(+), 29 deletions(-)

diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index a9be0ae2..ad37544a 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -27,12 +27,14 @@ __license__ = "GPL"
 from pickle import dumps, loads, HIGHEST_PROTOCOL
 import asynchat
 import asyncore
+import errno
 import fcntl
 import os
 import socket
 import sys
 import traceback
 
+from .utils import Utils
 from ..protocol import CSPROTO
 from ..helpers import getLogger,formatExceptionInfo
 
@@ -89,6 +91,29 @@ class RequestHandler(asynchat.async_chat):
 		self.close()
 		
 
+def loop(active, timeout=None, use_poll=False):
+		# Use poll instead of loop, because of recognition of active flag, 
+		# because of loop timeout mistake: different in poll and poll2 (sec vs ms),
+		# and to prevent sporadical errors like EBADF 'Bad file descriptor' etc. (see gh-161)
+		if timeout is None:
+			timeout = Utils.DEFAULT_SLEEP_TIME
+		poll = asyncore.poll
+		if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover
+			logSys.debug('Server listener (select) uses poll')
+			# poll2 expected a timeout in milliseconds (but poll and loop in seconds):
+			timeout = float(timeout) / 1000
+			poll = asyncore.poll2
+		# Poll as long as active:
+		while active():
+			try:
+				poll(timeout)
+			except Exception as e: # pragma: no cover
+				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor')
+					logSys.info('Server connection was closed: %s', str(e))
+				else:
+					logSys.error('Server connection was closed: %s', str(e))
+
+
 ##
 # Asynchronous server class.
 #
@@ -102,6 +127,7 @@ class AsyncServer(asyncore.dispatcher):
 		self.__transmitter = transmitter
 		self.__sock = "/var/run/fail2ban/fail2ban.sock"
 		self.__init = False
+		self.__active = False
 
 	##
 	# Returns False as we only read the socket first.
@@ -129,7 +155,7 @@ class AsyncServer(asyncore.dispatcher):
 	# @param sock: socket file.
 	# @param force: remove the socket file if exists.
 	
-	def start(self, sock, force):
+	def start(self, sock, force, use_poll=False):
 		self.__sock = sock
 		# Remove socket
 		if os.path.exists(sock):
@@ -149,28 +175,31 @@ class AsyncServer(asyncore.dispatcher):
 		AsyncServer.__markCloseOnExec(self.socket)
 		self.listen(1)
 		# Sets the init flag.
-		self.__init = True
-		# TODO Add try..catch
-		# There's a bug report for Python 2.6/3.0 that use_poll=True yields some 2.5 incompatibilities:
-		if (sys.version_info >= (2, 7) and sys.version_info < (2, 8)) \
-		   or (sys.version_info >= (3, 4)): # if python 2.7 ...
-			logSys.debug("Detected Python 2.7. asyncore.loop() using poll")
-			asyncore.loop(use_poll=True) # workaround for the "Bad file descriptor" issue on Python 2.7, gh-161
-		else:
-			asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0
-	
+		self.__init = self.__active = True
+		# Event loop as long as active:
+		loop(lambda: self.__active)
+		# Cleanup all
+		self.stop()
+
+
+	def close(self):
+		if self.__active:
+			asyncore.dispatcher.close(self)
+		# Remove socket (file) only if it was created:
+		if self.__init and os.path.exists(self.__sock):
+			logSys.debug("Removed socket file " + self.__sock)
+			os.remove(self.__sock)
+		logSys.debug("Socket shutdown")
+		self.__active = False
+
 	##
 	# Stops the communication server.
 	
 	def stop(self):
-		if self.__init:
-			# Only closes the socket if it was initialized first.
-			self.close()
-		# Remove socket
-		if os.path.exists(self.__sock):
-			logSys.debug("Removed socket file " + self.__sock)
-			os.remove(self.__sock)
-		logSys.debug("Socket shutdown")
+		self.close()
+
+	def isActive(self):
+		return self.__active
 
 	##
 	# Marks socket as close-on-exec to avoid leaking file descriptors when
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index d1ac3680..8dbb6a7a 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1023,8 +1023,8 @@ class DNSUtils:
 	@staticmethod
 	def ipToName(ip):
 		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
-		v = DNSUtils.CACHE_ipToName.get(ip)
-		if v is not None: 
+		v = DNSUtils.CACHE_ipToName.get(ip, ())
+		if v != ():
 			return v
 		# retrieve name
 		try:
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 27442832..37fe0138 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -19,7 +19,6 @@
 
 import os
 import smtpd
-import asyncore
 import threading
 import unittest
 import sys
@@ -30,7 +29,7 @@ else:
 
 from ..dummyjail import DummyJail
 
-from ..utils import CONFIG_DIR
+from ..utils import CONFIG_DIR, asyncserver
 
 
 class TestSMTPServer(smtpd.SMTPServer):
@@ -46,8 +45,6 @@ class SMTPActionTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
-		unittest.F2B.SkipIfNoNetwork()
-
 		self.jail = DummyJail()
 		pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")
 		pythonModuleName = os.path.basename(pythonModule.rstrip(".py"))
@@ -64,13 +61,16 @@ class SMTPActionTest(unittest.TestCase):
 		self.action = customActionModule.Action(
 			self.jail, "test", host="127.0.0.1:%i" % port)
 
+		## because of bug in loop (see loop in asyncserver.py) use it's loop instead of asyncore.loop:
+		self._active = True
 		self._loop_thread = threading.Thread(
-			target=asyncore.loop, kwargs={'timeout': 1})
+			target=asyncserver.loop, kwargs={'active': lambda: self._active})
 		self._loop_thread.start()
 
 	def tearDown(self):
 		"""Call after every test case."""
 		self.smtpd.close()
+		self._active = False
 		self._loop_thread.join()
 
 	def testStart(self):
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d26d7562..87ca4289 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -62,11 +62,14 @@ class TransmitterBase(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		self.transm = self.server._Server__transm
+		self.tmp_files = []
 		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter')
 		os.close(sock_fd)
+		self.tmp_files.append(sock_name)
 		pidfile_fd, pidfile_name = tempfile.mkstemp(
 			'fail2ban.pid', 'transmitter')
 		os.close(pidfile_fd)
+		self.tmp_files.append(pidfile_name)
 		self.server.start(sock_name, pidfile_name, force=False)
 		self.jailName = "TestJail1"
 		self.server.addJail(self.jailName, "auto")
@@ -74,6 +77,9 @@ class TransmitterBase(unittest.TestCase):
 	def tearDown(self):
 		"""Call after every test case."""
 		self.server.quit()
+		for f in self.tmp_files:
+			if os.path.exists(f):
+				os.remove(f)
 
 	def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False):
 		"""Process set/get commands and compare both return values 
diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py
index a9408fde..5bf0be57 100644
--- a/fail2ban/tests/sockettestcase.py
+++ b/fail2ban/tests/sockettestcase.py
@@ -55,6 +55,24 @@ class Socket(unittest.TestCase):
 		"""Test transmitter proceed method which just returns first arg"""
 		return message
 
+	def testStopPerCloseUnexpected(self):
+		# start in separate thread :
+		serverThread = threading.Thread(
+			target=self.server.start, args=(self.sock_name, False))
+		serverThread.daemon = True
+		serverThread.start()
+		self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10)))
+		# unexpected stop directly after start:
+		self.server.close()
+		# wait for end of thread :
+		Utils.wait_for(lambda: not serverThread.isAlive() 
+			or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10))
+		self.assertFalse(serverThread.isAlive())
+		# clean :
+		self.server.stop()
+		self.assertFalse(self.server.isActive())
+		self.assertFalse(os.path.exists(self.sock_name))
+
 	def _serverSocket(self):
 		try:
 			return CSocket(self.sock_name)
@@ -66,6 +84,7 @@ class Socket(unittest.TestCase):
 			target=self.server.start, args=(self.sock_name, False))
 		serverThread.daemon = True
 		serverThread.start()
+		self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10)))
 		time.sleep(Utils.DEFAULT_SLEEP_TIME)
 
 		client = Utils.wait_for(self._serverSocket, 2)
@@ -78,7 +97,11 @@ class Socket(unittest.TestCase):
 		client.close()
 
 		self.server.stop()
-		serverThread.join(Utils.DEFAULT_SLEEP_TIME)
+		# wait for end of thread :
+		Utils.wait_for(lambda: not serverThread.isAlive() 
+			or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10))
+		self.assertFalse(serverThread.isAlive())
+		self.assertFalse(self.server.isActive())
 		self.assertFalse(os.path.exists(self.sock_name))
 
 	def testSocketForce(self):
@@ -92,10 +115,13 @@ class Socket(unittest.TestCase):
 			target=self.server.start, args=(self.sock_name, True))
 		serverThread.daemon = True
 		serverThread.start()
-		time.sleep(Utils.DEFAULT_SLEEP_TIME)
+		self.assertTrue(Utils.wait_for(self.server.isActive, unittest.F2B.maxWaitTime(10)))
 
 		self.server.stop()
-		serverThread.join(Utils.DEFAULT_SLEEP_TIME)
+		# wait for end of thread :
+		Utils.wait_for(lambda: not serverThread.isAlive() 
+			or serverThread.join(Utils.DEFAULT_SLEEP_INTERVAL), unittest.F2B.maxWaitTime(10))
+		self.assertFalse(self.server.isActive())
 		self.assertFalse(os.path.exists(self.sock_name))
 
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 338ccec1..eb4e6e2e 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -35,6 +35,8 @@ from ..helpers import getLogger
 from ..server.filter import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
+# for action_d.test_smtp :
+from ..server import asyncserver
 
 
 logSys = getLogger(__name__)
@@ -61,6 +63,10 @@ class F2B(optparse.Values):
 		pass
 	def SkipIfNoNetwork(self):
 		pass
+	def maxWaitTime(self,wtime):
+		if self.fast:
+			wtime = float(wtime) / 10
+		return wtime
 
 
 def initTests(opts):
@@ -87,6 +93,13 @@ def initTests(opts):
 		def F2B_SkipIfNoNetwork():
 			raise unittest.SkipTest('Skip test because of "--no-network"')
 		unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork
+	# precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737):
+	c = DNSUtils.CACHE_ipToName
+	for i in xrange(255):
+		c.set('192.0.2.%s' % i, None)
+		c.set('198.51.100.%s' % i, None)
+		c.set('203.0.113.%s' % i, None)
+
 
 def mtimesleep():
 	# no sleep now should be necessary since polling tracks now not only

From f143ae479cee8510ddc7385cb84228a4823b58aa Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 1 Sep 2015 20:10:47 +0200
Subject: [PATCH 018/623] code and config review after rebase to current master

---
 fail2ban/server/server.py        | 5 ++---
 fail2ban/tests/filtertestcase.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 86197bc5..3bdfd71b 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -326,9 +326,8 @@ class Server:
 	def isAlive(self, jailnum=None):
 		if jailnum is not None and len(self.__jails) != jailnum:
 			return 0
-		for j in self.__jails:
-			j = self.__jails[j]
-			if not j.isAlive():
+		for jail in self.__jails.values():
+			if not jail.isAlive():
 				return 0
 		return 1
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 11642d2c..5edfd267 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1302,9 +1302,9 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 				self.assertEqual(res, [])
 
 	def testIpToName(self):
+		unittest.F2B.SkipIfNoNetwork()
 		res = DNSUtils.ipToName('8.8.4.4')
 		self.assertEqual(res, 'google-public-dns-b.google.com')
-		unittest.F2B.SkipIfNoNetwork()
 		# invalid ip (TEST-NET-1 according to RFC 5737)
 		res = DNSUtils.ipToName('192.0.2.0')
 		self.assertEqual(res, None)

From f7cc55103c77cd4dc0502268ea05215c32a502c6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 Nov 2015 08:56:41 +0100
Subject: [PATCH 019/623] optimized FailManager: increase performance, try to
 prevent memory leakage (don't copy failures resp. it list on some operations)

---
 fail2ban/server/failmanager.py        | 87 +++++++++++----------------
 fail2ban/tests/failmanagertestcase.py |  4 +-
 2 files changed, 36 insertions(+), 55 deletions(-)

diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 4d006098..905553d4 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -44,53 +44,30 @@ class FailManager:
 		self.__failTotal = 0
 	
 	def setFailTotal(self, value):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			self.__failTotal = value
-		finally:
-			self.__lock.release()
 		
 	def getFailTotal(self):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			return self.__failTotal
-		finally:
-			self.__lock.release()
 	
 	def setMaxRetry(self, value):
-		try:
-			self.__lock.acquire()
-			self.__maxRetry = value
-		finally:
-			self.__lock.release()
+		self.__maxRetry = value
 	
 	def getMaxRetry(self):
-		try:
-			self.__lock.acquire()
-			return self.__maxRetry
-		finally:
-			self.__lock.release()
+		return self.__maxRetry
 	
 	def setMaxTime(self, value):
-		try:
-			self.__lock.acquire()
-			self.__maxTime = value
-		finally:
-			self.__lock.release()
+		self.__maxTime = value
 	
 	def getMaxTime(self):
-		try:
-			self.__lock.acquire()
-			return self.__maxTime
-		finally:
-			self.__lock.release()
+		return self.__maxTime
 
 	def addFailure(self, ticket, count=1):
 		attempts = 1
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			ip = ticket.getIP()
-			if ip in self.__failList:
+			try:
 				fData = self.__failList[ip]
 				# if the same object:
 				if fData is ticket:
@@ -103,7 +80,7 @@ class FailManager:
 					fData.setRetry(0)
 				fData.inc(matches, 1, count)
 				fData.setLastTime(unixTime)
-			else:
+			except KeyError:
 				# if already FailTicket - add it direct, otherwise create (using copy all ticket data):
 				if isinstance(ticket, FailTicket):
 					fData = ticket;
@@ -124,42 +101,46 @@ class FailManager:
 											  for k,v in  self.__failList.iteritems()])
 				logSys.debug("Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
 							 % (self.__failTotal, len(self.__failList), failures_summary))
-		finally:
-			self.__lock.release()
 		return attempts
 	
 	def size(self):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			return len(self.__failList)
-		finally:
-			self.__lock.release()
 	
 	def cleanup(self, time):
-		try:
-			self.__lock.acquire()
-			tmp = self.__failList.copy()
-			for item in tmp:
-				if tmp[item].getLastTime() < time - self.__maxTime:
-					self.__delFailure(item)
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			todelete = [ip for ip,item in self.__failList.iteritems() \
+				if item.getLastTime() + self.__maxTime <= time]
+			if len(todelete) == len(self.__failList):
+				# remove all:
+				self.__failList = dict()
+			elif not len(todelete):
+				# nothing:
+				return
+			if len(todelete) / 2.0 <= len(self.__failList) / 3.0:
+				# few as 2/3 should be removed - remove particular items:
+				for ip in todelete:
+					del self.__failList[ip]
+			else:
+				# create new dictionary without items to be deleted:
+				self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \
+					if item.getLastTime() + self.__maxTime > time)
 	
-	def __delFailure(self, ip):
-		if ip in self.__failList:
-			del self.__failList[ip]
+	def delFailure(self, ip):
+		with self.__lock:
+			try:
+				del self.__failList[ip]
+			except KeyError:
+				pass
 	
 	def toBan(self, ip=None):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			for ip in ([ip] if ip != None and ip in self.__failList else self.__failList):
 				data = self.__failList[ip]
 				if data.getRetry() >= self.__maxRetry:
 					del self.__failList[ip]
 					return data
 			raise FailManagerEmpty
-		finally:
-			self.__lock.release()
 
 
 class FailManagerEmpty(Exception):
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index a8c0f6fc..5a6e95e7 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -68,11 +68,11 @@ class AddFailure(unittest.TestCase):
 		self.assertEqual(self.__failManager.getMaxTime(), 13)
 		self.__failManager.setMaxTime(600)
 
-	def _testDel(self):
+	def testDel(self):
 		self.__failManager.delFailure('193.168.0.128')
 		self.__failManager.delFailure('111.111.1.111')
 		
-		self.assertEqual(self.__failManager.size(), 1)
+		self.assertEqual(self.__failManager.size(), 2)
 		
 	def testCleanupOK(self):
 		timestamp = 1167606999.0

From a10eb39bbef646c5d1790db16882973028f0f695 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Thu, 19 Nov 2015 04:26:35 +0100
Subject: [PATCH 020/623] test cases extended with memory leakage check

---
 fail2ban/server/failmanager.py        |   5 +-
 fail2ban/tests/failmanagertestcase.py | 105 ++++++++++++++++++++++++++
 fail2ban/tests/utils.py               |   1 +
 3 files changed, 109 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 905553d4..07be0a4c 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -32,6 +32,7 @@ from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
+logLevel = logging.DEBUG
 
 
 class FailManager:
@@ -93,13 +94,13 @@ class FailManager:
 			attempts = fData.getRetry()
 			self.__failTotal += 1
 
-			if logSys.getEffectiveLevel() <= logging.DEBUG:
+			if logSys.getEffectiveLevel() <= logLevel:
 				# yoh: Since composing this list might be somewhat time consuming
 				# in case of having many active failures, it should be ran only
 				# if debug level is "low" enough
 				failures_summary = ', '.join(['%s:%d' % (k, v.getRetry())
 											  for k,v in  self.__failList.iteritems()])
-				logSys.debug("Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
+				logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
 							 % (self.__failTotal, len(self.__failList), failures_summary))
 		return attempts
 	
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 5a6e95e7..2d4ce430 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -26,6 +26,7 @@ __license__ = "GPL"
 
 import unittest
 
+from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
 from ..server.ticket import FailTicket
 
@@ -120,3 +121,107 @@ class AddFailure(unittest.TestCase):
 		ticket = self.__failManager.toBan()
 		self.assertNotEqual(ticket.getIP(), "100.100.10.10")
 		self.assertRaises(FailManagerEmpty, self.__failManager.toBan)
+
+
+class FailmanagerComplex(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		super(FailmanagerComplex, self).setUp()
+		self.__failManager = FailManager()
+		# down logging level for all this tests, because of extremely large failure count (several GB on heavydebug)
+		self.__saved_ll = failmanager.logLevel
+		failmanager.logLevel = 3
+
+	def tearDown(self):
+		super(FailmanagerComplex, self).tearDown()
+		# restore level
+		failmanager.logLevel = self.__saved_ll
+
+	@staticmethod
+	def _ip_range(maxips):
+		class _ip(list):
+			def __str__(self):
+				return '.'.join(map(str, self))
+			def __repr__(self):
+				return str(self)
+			def __key__(self):
+				return str(self)
+			def __hash__(self):
+				#return (int)(struct.unpack('I', struct.pack("BBBB",*self))[0])
+				return (int)(self[0] << 24 | self[1] << 16 | self[2] << 8 | self[3])
+		i = 0
+		c = [127,0,0,0]
+		while i < maxips:
+			for n in range(3,0,-1):
+				if c[n] < 255:
+					c[n] += 1
+					break
+				c[n] = 0
+			yield (i, _ip(c))
+			i += 1
+
+	def testCheckIPGenerator(self):
+		for i, ip in self._ip_range(65536 if not unittest.F2B.fast else 1000):
+			if i == 254:
+				self.assertEqual(str(ip), '127.0.0.255')
+			elif i == 255:
+				self.assertEqual(str(ip), '127.0.1.0')
+			elif i == 1000:
+				self.assertEqual(str(ip), '127.0.3.233')
+			elif i == 65534:
+				self.assertEqual(str(ip), '127.0.255.255')
+			elif i == 65535:
+				self.assertEqual(str(ip), '127.1.0.0')
+	
+	def testFailuresMemLeak1(self):
+		# use factor (divisor) instead unittest.F2B.SkipIfFast() :
+		modeDiv = 1 
+		if unittest.F2B.fast: # pragma: no cover
+			modeDiv = 10
+			self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv)
+		import gc
+		gc.collect()
+		timestamp = 1167606999.0
+		ticktime = timestamp-(1000 // modeDiv)
+		for i, ip in self._ip_range(30000 // modeDiv):
+			t = FailTicket(ip, ticktime, list({'match': i}))
+			ticktime += 1
+			self.__failManager.addFailure(t)
+			if i % (500 // modeDiv) == 0:
+			 	self.__failManager.cleanup(timestamp)
+			if i % (888 // modeDiv) == 0:
+				timestamp += (1000 // modeDiv)
+				ticktime = timestamp-(1000 // modeDiv)
+		self.assertFalse(gc.collect())
+		self.__failManager.cleanup(timestamp)
+		ticktime -= timestamp - self.__failManager.getMaxTime() + 1
+		self.assertEqual(self.__failManager.size(), ticktime if ticktime > 0 else 0)
+		self.assertFalse(gc.collect())
+
+	def testFailuresMemLeak2(self):
+		# use factor (divisor) instead unittest.F2B.SkipIfFast() :
+		modeDiv = 1 
+		if unittest.F2B.fast: # pragma: no cover
+			modeDiv = 10
+			self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv)
+		import gc
+		gc.collect()
+		timestamp = 1167606999.0
+		ticktime = timestamp-(1000 // modeDiv)
+		for i, ip in self._ip_range(10000 // modeDiv):
+			for j in range(0, 5):
+				t = FailTicket(ip, ticktime, list({'match': i}))
+				ticktime += 1
+				self.__failManager.addFailure(t)
+			if i % (500 // modeDiv) == 0:
+			 	self.__failManager.cleanup(timestamp)
+			if i % (888 // modeDiv) == 0 or i % (1500 // modeDiv) == 0:
+				timestamp += (1000 // modeDiv)
+				ticktime = timestamp-(1000 // modeDiv)
+		self.assertFalse(gc.collect())
+		timestamp += 20000
+		self.__failManager.cleanup(timestamp)
+		self.assertEqual(self.__failManager.size(), 0)
+		self.assertFalse(gc.collect())
+
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index eb4e6e2e..c33c2b19 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -176,6 +176,7 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(tickettestcase.TicketTests))
 	# FailManager
 	tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
+	tests.addTest(unittest.makeSuite(failmanagertestcase.FailmanagerComplex))
 	# BanManager
 	tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure))
 	try:

From 6406f6f5600deaf1d19f793cef8e30a1f55bf43f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 23 Nov 2015 12:48:07 +0100
Subject: [PATCH 021/623] background servicing (temporally executed from
 failmanager): prevents memory leak on some platforms/python versions, using
 forced GC in periodic intervals (latency and threshold); Side effect: GC is
 disabled now inside fail2ban-server (to avoid multiple garbage collect)

---
 fail2ban/helpers.py                   | 58 +++++++++++++++++++++++++--
 fail2ban/server/failmanager.py        |  9 ++++-
 fail2ban/tests/failmanagertestcase.py | 23 +++++++++++
 3 files changed, 84 insertions(+), 6 deletions(-)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 8e1b0e32..b5ffe601 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -20,11 +20,16 @@
 __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko"
 __license__ = "GPL"
 
-import sys
-import os
-import traceback
-import re
+import gc
 import logging
+import os
+import re
+import sys
+import traceback
+
+from threading import Lock
+
+from .server.mytime import MyTime
 
 
 def formatExceptionInfo():
@@ -137,3 +142,48 @@ def splitcommaspace(s):
 	if not s:
 		return []
 	return filter(bool, re.split('[ ,]', s))
+
+
+class BgService(object):
+	"""Background servicing
+
+	Prevents memory leak on some platforms/python versions, 
+	using forced GC in periodical intervals.
+	"""
+
+	_mutex = Lock()
+	_instance = None
+	def __new__(cls):
+		if not cls._instance:
+			cls._instance = \
+				super(BgService, cls).__new__(cls)
+		return cls._instance
+
+	def __init__(self):
+		self.__serviceTime = -0x7fffffff
+		self.__periodTime = 30
+		self.__threshold = 100;
+		self.__count = self.__threshold;
+		if hasattr(gc, 'set_threshold'):
+			gc.set_threshold(0)
+		gc.disable()
+
+	def service(self, force=False, wait=False):
+		self.__count -= 1
+		# avoid locking if next service time don't reached
+		if not force and (self.__count > 0 or MyTime.time() < self.__serviceTime):
+			return False
+		# return immediately if mutex already locked (other thread in servicing):
+		if not BgService._mutex.acquire(wait):
+			return False
+		try:
+			# check again in lock:
+			if MyTime.time() < self.__serviceTime:
+				return False
+			gc.collect()
+			self.__serviceTime = MyTime.time() + self.__periodTime
+			self.__count = self.__threshold
+			return True
+		finally:
+			BgService._mutex.release()
+		return False
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 07be0a4c..45f3a393 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -28,7 +28,7 @@ from threading import Lock
 import logging
 
 from .ticket import FailTicket
-from ..helpers import getLogger
+from ..helpers import getLogger, BgService
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -43,6 +43,7 @@ class FailManager:
 		self.__maxRetry = 3
 		self.__maxTime = 600
 		self.__failTotal = 0
+		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
 		with self.__lock:
@@ -102,6 +103,8 @@ class FailManager:
 											  for k,v in  self.__failList.iteritems()])
 				logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
 							 % (self.__failTotal, len(self.__failList), failures_summary))
+
+		self.__bgSvc.service()
 		return attempts
 	
 	def size(self):
@@ -126,6 +129,7 @@ class FailManager:
 				# create new dictionary without items to be deleted:
 				self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \
 					if item.getLastTime() + self.__maxTime > time)
+		self.__bgSvc.service()
 	
 	def delFailure(self, ip):
 		with self.__lock:
@@ -141,7 +145,8 @@ class FailManager:
 				if data.getRetry() >= self.__maxRetry:
 					del self.__failList[ip]
 					return data
-			raise FailManagerEmpty
+		self.__bgSvc.service()
+		raise FailManagerEmpty
 
 
 class FailManagerEmpty(Exception):
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 2d4ce430..78f7b509 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -122,6 +122,29 @@ class AddFailure(unittest.TestCase):
 		self.assertNotEqual(ticket.getIP(), "100.100.10.10")
 		self.assertRaises(FailManagerEmpty, self.__failManager.toBan)
 
+	def testBgService(self):
+		bgSvc = self.__failManager._FailManager__bgSvc
+		failManager2nd = FailManager()
+		# test singleton (same object):
+		bgSvc2 = failManager2nd._FailManager__bgSvc
+		self.assertTrue(id(bgSvc) == id(bgSvc2))
+		bgSvc2 = None
+		# test service :
+		self.assertTrue(bgSvc.service(True, True))
+		self.assertFalse(bgSvc.service())
+		# bypass threshold and time:
+		for i in range(1, bgSvc._BgService__threshold):
+			self.assertFalse(bgSvc.service())
+		# bypass time check:
+		bgSvc._BgService__serviceTime = -0x7fffffff
+		self.assertTrue(bgSvc.service())
+		# bypass threshold and time:
+		bgSvc._BgService__serviceTime = -0x7fffffff
+		for i in range(1, bgSvc._BgService__threshold):
+			self.assertFalse(bgSvc.service())
+		self.assertTrue(bgSvc.service(False, True))
+		self.assertFalse(bgSvc.service(False, True))
+
 
 class FailmanagerComplex(unittest.TestCase):
 

From 935d79eaaedef532230ab72a8256be713b27de1f Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 29 Nov 2015 01:04:55 +0100
Subject: [PATCH 022/623] 1) prevents a bug by logging stdout/stderr if retcode
 still None: ``` in executeCmd     if retcode < 0: TypeError: unorderable
 types: NoneType() < int() ``` 2) prevents a rarely test case bug of
 testExecuteTimeoutWithNastyChildren, because no stdout (Resource temporarily
 unavailable), possible no flush by IO of the killing process;

---
 fail2ban/server/utils.py         | 4 ++--
 fail2ban/tests/actiontestcase.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 262b303d..45d1c09d 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -159,7 +159,7 @@ class Utils():
 			# if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
 			if popen.stdout:
 				try:
-					if retcode < 0:
+					if retcode is None or retcode < 0:
 						Utils.setFBlockMode(popen.stdout, False)
 					stdout = popen.stdout.read()
 				except IOError as e:
@@ -169,7 +169,7 @@ class Utils():
 				popen.stdout.close()
 			if popen.stderr:
 				try:
-					if retcode < 0:
+					if retcode is None or retcode < 0:
 						Utils.setFBlockMode(popen.stderr, False)
 					stderr = popen.stderr.read()
 				except IOError as e:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 8c9b5ef1..6d8fcc82 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -247,7 +247,7 @@ class CommandActionTest(LogCaptureTestCase):
 		cpid = getnastypid()
 		# Verify that the process itself got killed
 		self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3))  # process should have been killed
-		self.assertLogged('my pid ')
+		self.assertLogged('my pid ', 'Resource temporarily unavailable')
 		self.assertLogged('timed out')
 		self.assertLogged('killed with SIGTERM', 
 		                  'killed with SIGKILL')
@@ -261,7 +261,7 @@ class CommandActionTest(LogCaptureTestCase):
 		cpid = getnastypid()
 		# Verify that the process itself got killed
 		self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3))
-		self.assertLogged('my pid ')
+		self.assertLogged('my pid ', 'Resource temporarily unavailable')
 		self.assertLogged('timed out')
 		self.assertLogged('killed with SIGTERM', 
 		                  'killed with SIGKILL')

From e065941ac52c2566595d9dfa29f07e5fa419cf1c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Dec 2015 21:35:37 +0100
Subject: [PATCH 023/623] use "maxEntries" (currently 50 as default) as range
 for max number of the last matches/failures, fail2ban will hold per IP in the
 list of failures in failmanager resp. in the database; prevents out of memory
 situation if many IP's makes extremely many failures (or very large files
 since last fail2ban run); closes gh-1277 todo: parameter `maxentries` should
 be configurable (jail.conf resp. fail2ban.conf); todo: adjust ban-time-incr
 branch by merge (table "bips").

---
 fail2ban/server/database.py           | 12 +++++-
 fail2ban/server/failmanager.py        | 16 +++++--
 fail2ban/tests/databasetestcase.py    | 31 ++++++++++++++
 fail2ban/tests/failmanagertestcase.py | 62 ++++++++++++++++++++++++---
 4 files changed, 111 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 3b419ed3..b7fd4d47 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -163,6 +163,7 @@ class Fail2BanDb(object):
 
 
 	def __init__(self, filename, purgeAge=24*60*60):
+		self.maxEntries = 50
 		try:
 			self._lock = RLock()
 			self._db = sqlite3.connect(
@@ -454,7 +455,7 @@ class Fail2BanDb(object):
 		if ip is not None:
 			query += " AND ip=?"
 			queryArgs.append(ip)
-		query += " ORDER BY ip, timeofban"
+		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)
 
@@ -535,7 +536,14 @@ class Fail2BanDb(object):
 						matches = []
 						failures = 0
 						tickdata = {}
-					matches.extend(data.get('matches', ()))
+					m = data.get('matches', [])
+					# pre-insert "maxadd" enries (because tickets are ordered desc by time)
+					maxadd = self.maxEntries - len(matches)
+					if maxadd > 0:
+						if len(m) <= maxadd:
+							matches = m + matches
+						else:
+							matches = m[-maxadd:] + matches
 					failures += data.get('failures', 1)
 					tickdata.update(data.get('data', {}))
 					prev_timeofban = timeofban
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index 45f3a393..ae97b36a 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -43,6 +43,7 @@ class FailManager:
 		self.__maxRetry = 3
 		self.__maxTime = 600
 		self.__failTotal = 0
+		self.maxEntries = 50
 		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
@@ -71,17 +72,26 @@ class FailManager:
 			ip = ticket.getIP()
 			try:
 				fData = self.__failList[ip]
-				# if the same object:
+				# if the same object - the same matches but +1 attempt:
 				if fData is ticket:
 					matches = None
+					attempt = 1
 				else:
+					# will be incremented / extended (be sure we have at least +1 attempt):
 					matches = ticket.getMatches()
+					attempt = ticket.getAttempt()
+					if attempt <= 0:
+						attempt += 1
 				unixTime = ticket.getTime()
+				fData.setLastTime(unixTime)
 				if fData.getLastReset() < unixTime - self.__maxTime:
 					fData.setLastReset(unixTime)
 					fData.setRetry(0)
-				fData.inc(matches, 1, count)
-				fData.setLastTime(unixTime)
+				fData.inc(matches, attempt, count)
+				# truncate to maxEntries:
+				matches = fData.getMatches()
+				if len(matches) > self.maxEntries:
+					fData.setMatches(matches[-self.maxEntries:])
 			except KeyError:
 				# if already FailTicket - add it direct, otherwise create (using copy all ticket data):
 				if isinstance(ticket, FailTicket):
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 6b85af56..5d83710f 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -251,6 +251,37 @@ class DatabaseTest(LogCaptureTestCase):
 		# be returned
 		self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=-1)), 2)
 
+	def testGetBansMerged_MaxEntries(self):
+		if Fail2BanDb is None: # pragma: no cover
+			return
+		self.testAddJail()
+		maxEntries = 2
+		failures = ["abc\n", "123\n", "ABC\n", "1234\n"]
+		# add failures sequential:
+		i = 80
+		for f in failures:
+			i -= 10
+			ticket = FailTicket("127.0.0.1", MyTime.time() - i, [f])
+			ticket.setAttempt(1)
+			self.db.addBan(self.jail, ticket)
+		# should retrieve 2 matches only, but count of all attempts:
+		self.db.maxEntries = maxEntries;
+		ticket = self.db.getBansMerged("127.0.0.1")
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+    # add more failures at once:
+		ticket = FailTicket("127.0.0.1", MyTime.time() - 10, failures)
+		ticket.setAttempt(len(failures))
+		self.db.addBan(self.jail, ticket)
+		# should retrieve 2 matches only, but count of all attempts:
+		self.db.maxEntries = maxEntries;
+		ticket = self.db.getBansMerged("127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), 2 * len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+
 	def testGetBansMerged(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 78f7b509..3cac0540 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -35,6 +35,13 @@ class AddFailure(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		self.__items = None
+		self.__failManager = FailManager()
+
+	def tearDown(self):
+		"""Call after every test case."""
+
+	def _addDefItems(self):
 		self.__items = [[u'193.168.0.128', 1167605999.0],
 					    [u'193.168.0.128', 1167605999.0],
 					    [u'193.168.0.128', 1167605999.0],
@@ -48,44 +55,87 @@ class AddFailure(unittest.TestCase):
 					    ['100.100.10.10', 1000001000.0],
 					    ['100.100.10.10', 1000001500.0],
 					    ['100.100.10.10', 1000002000.0]]
-		
-		self.__failManager = FailManager()
 		for i in self.__items:
 			self.__failManager.addFailure(FailTicket(i[0], i[1]))
 
-	def tearDown(self):
-		"""Call after every test case."""
-	
 	def testFailManagerAdd(self):
+		self._addDefItems()
 		self.assertEqual(self.__failManager.size(), 3)
 		self.assertEqual(self.__failManager.getFailTotal(), 13)
 		self.__failManager.setFailTotal(0)
 		self.assertEqual(self.__failManager.getFailTotal(), 0)
 		self.__failManager.setFailTotal(13)
 	
+	def testFailManagerAdd_MaxEntries(self):
+		maxEntries = 2
+		self.__failManager.maxEntries = maxEntries
+		failures = ["abc\n", "123\n", "ABC\n", "1234\n"]
+		# add failures sequential:
+		i = 80
+		for f in failures:
+			i -= 10
+			ticket = FailTicket("127.0.0.1", 1000002000 - i, [f])
+			ticket.setAttempt(1)
+			self.__failManager.addFailure(ticket)
+		#
+		manFailList = self.__failManager._FailManager__failList
+		self.assertEqual(len(manFailList), 1)
+		ticket = manFailList["127.0.0.1"]
+		# should retrieve 2 matches only, but count of all attempts (4):
+		self.assertEqual(ticket.getAttempt(), len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+    # add more failures at once:
+		ticket = FailTicket("127.0.0.1", 1000002000 - 10, failures)
+		ticket.setAttempt(len(failures))
+		self.__failManager.addFailure(ticket)
+		#
+		manFailList = self.__failManager._FailManager__failList
+		self.assertEqual(len(manFailList), 1)
+		ticket = manFailList["127.0.0.1"]
+		# should retrieve 2 matches only, but count of all attempts (8):
+		self.assertEqual(ticket.getAttempt(), 2 * len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+		# add self ticket again:
+		self.__failManager.addFailure(ticket)
+		#
+		manFailList = self.__failManager._FailManager__failList
+		self.assertEqual(len(manFailList), 1)
+		ticket = manFailList["127.0.0.1"]
+		# same matches, but +1 attempt (9)
+		self.assertEqual(ticket.getAttempt(), 2 * len(failures) + 1)
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+	
 	def testFailManagerMaxTime(self):
+		self._addDefItems()
 		self.assertEqual(self.__failManager.getMaxTime(), 600)
 		self.__failManager.setMaxTime(13)
 		self.assertEqual(self.__failManager.getMaxTime(), 13)
 		self.__failManager.setMaxTime(600)
 
 	def testDel(self):
+		self._addDefItems()
 		self.__failManager.delFailure('193.168.0.128')
 		self.__failManager.delFailure('111.111.1.111')
 		
 		self.assertEqual(self.__failManager.size(), 2)
 		
 	def testCleanupOK(self):
+		self._addDefItems()
 		timestamp = 1167606999.0
 		self.__failManager.cleanup(timestamp)
 		self.assertEqual(self.__failManager.size(), 0)
 		
 	def testCleanupNOK(self):
+		self._addDefItems()
 		timestamp = 1167605990.0
 		self.__failManager.cleanup(timestamp)
 		self.assertEqual(self.__failManager.size(), 2)
 	
 	def testbanOK(self):
+		self._addDefItems()
 		self.__failManager.setMaxRetry(5)
 		#ticket = FailTicket('193.168.0.128', None)
 		ticket = self.__failManager.toBan()
@@ -112,10 +162,12 @@ class AddFailure(unittest.TestCase):
 			'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5 matches=[]')
 	
 	def testbanNOK(self):
+		self._addDefItems()
 		self.__failManager.setMaxRetry(10)
 		self.assertRaises(FailManagerEmpty, self.__failManager.toBan)
 
 	def testWindow(self):
+		self._addDefItems()
 		ticket = self.__failManager.toBan()
 		self.assertNotEqual(ticket.getIP(), "100.100.10.10")
 		ticket = self.__failManager.toBan()

From 3f2b58e9730e0c743fd0db2eeacd34585b7fa36b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 23 Dec 2015 18:49:30 +0100
Subject: [PATCH 024/623] datedetector: matchTime, template list etc. rewritten
 because of performance degradation (without sorting templates now) - in-place
 reordering using hits and last used time; todo: rewrite "reGroupDictStrptime"
 because may be too slow;

---
 fail2ban/client/fail2banregex.py       |  4 +-
 fail2ban/server/datedetector.py        | 65 ++++++++++++++++----------
 fail2ban/server/datetemplate.py        |  1 +
 fail2ban/server/filtergamin.py         |  1 -
 fail2ban/server/filterpoll.py          |  1 -
 fail2ban/server/filterpyinotify.py     |  1 -
 fail2ban/tests/datedetectortestcase.py |  9 ----
 fail2ban/tests/filtertestcase.py       | 35 +++++++++++++-
 8 files changed, 75 insertions(+), 42 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index d0cdab84..1fc5316e 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -366,7 +366,7 @@ class Fail2banRegex(object):
 
 	def process(self, test_lines):
 		t0 = time.time()
-		for line_no, line in enumerate(test_lines):
+		for line in test_lines:
 			if isinstance(line, tuple):
 				line_datetimestripped, ret = self.testRegex(
 					line[0], line[1])
@@ -398,8 +398,6 @@ class Fail2banRegex(object):
 						self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
 			self._line_stats.tested += 1
 
-			if line_no % 10 == 0 and self._filter.dateDetector is not None:
-				self._filter.dateDetector.sortTemplate()
 		self._time_elapsed = time.time() - t0
 
 	def printLines(self, ltype):
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 4bdcefd2..4004d5c5 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -21,6 +21,8 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
+import time
+
 from threading import Lock
 
 from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch
@@ -44,6 +46,8 @@ class DateDetector(object):
 		self.__lock = Lock()
 		self.__templates = list()
 		self.__known_names = set()
+		# time the template was long unused (currently 300 == 5m):
+		self.__unusedTime = 300
 
 	def _appendTemplate(self, template):
 		name = template.name
@@ -153,22 +157,29 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
-		self.__lock.acquire()
-		try:
+		i = 0
+		with self.__lock:
 			for template in self.__templates:
 				match = template.matchDate(line)
 				if not match is None:
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "Matched time template %s", template.name)
 					template.hits += 1
+					template.lastUsed = time.time()
+					# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
+					if i:
+						self._reorderTemplate(i)
+					# return tuple with match and template reference used for parsing:
 					return (match, template)
-			return (None, None)
-		finally:
-			self.__lock.release()
+				i += 1
+		# not found:
+		return (None, None)
 
 	def getTime(self, line):
 		"""Attempts to return the date on a log line using templates.
 
+		Obsolete: Use "getTime2" instead.
+
 		This uses the templates' `getDate` method in an attempt to find
 		a date.
 
@@ -183,8 +194,7 @@ class DateDetector(object):
 			The Unix timestamp returned from the first successfully matched
 			template or None if not found.
 		"""
-		self.__lock.acquire()
-		try:
+		with self.__lock:
 			for template in self.__templates:
 				try:
 					date = template.getDate(line)
@@ -197,8 +207,6 @@ class DateDetector(object):
 				except ValueError: # pragma: no cover
 					pass
 			return None
-		finally:
-			self.__lock.release()
 
 	def getTime2(self, line, timeMatch = None):
 		"""Attempts to return the date on a log line using given template.
@@ -232,21 +240,28 @@ class DateDetector(object):
 			return date
 		return self.getTime(line)
 
-	def sortTemplate(self):
-		"""Sort the date templates by number of hits
+	def _reorderTemplate(self, num):
+		"""Reorder template (bubble up) in template list if hits grows enough.
 
-		Sort the template lists using the hits score. This method is not
-		called in this object and thus should be called from time to time.
-		This ensures the most commonly matched templates are checked first,
-		improving performance of matchTime and getTime.
+		Parameters
+		----------
+		num : int
+			Index of template should be moved.
 		"""
-		self.__lock.acquire()
-		try:
-			if logSys.getEffectiveLevel() <= logLevel:
-				logSys.log(logLevel, "Sorting the template list")
-			self.__templates.sort(key=lambda x: x.hits, reverse=True)
-			t = self.__templates[0]
-			if logSys.getEffectiveLevel() <= logLevel:
-				logSys.log(logLevel, "Winning template: %s with %d hits", t.name, t.hits)
-		finally:
-			self.__lock.release()
+		if num:
+			templates = self.__templates
+			template = templates[num]
+		  ## current hits and time the template was long unused:
+			untime = template.lastUsed - self.__unusedTime
+			hits = template.hits
+			## don't move too often (multiline logs resp. log's with different date patterns),
+			## if template not used too long, replace it also :
+			if hits > templates[num-1].hits + 5 or templates[num-1].lastUsed < untime:
+				## try to move faster (half of part to current template):
+				pos = num // 2
+				## if not larger - move slow (exact 1 position):
+				if hits <= templates[pos].hits or templates[pos].lastUsed < untime:
+					pos = num-1
+				templates[pos], templates[num] = template, templates[pos]
+
+
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index e1f6b3de..88e2ae3f 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -50,6 +50,7 @@ class DateTemplate(object):
 		self._regex = ""
 		self._cRegex = None
 		self.hits = 0
+		self.lastUsed = 0
 
 	@property
 	def name(self):
diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index b094cfeb..6b562a5c 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -84,7 +84,6 @@ class FilterGamin(FileFilter):
 				self.jail.putFailTicket(ticket)
 		except FailManagerEmpty:
 			self.failManager.cleanup(MyTime.time())
-		self.dateDetector.sortTemplate()
 		self.__modified = False
 
 	##
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index d7ff9075..c7b04970 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -118,7 +118,6 @@ class FilterPoll(FileFilter):
 						self.jail.putFailTicket(ticket)
 				except FailManagerEmpty:
 					self.failManager.cleanup(MyTime.time())
-				self.dateDetector.sortTemplate()
 				self.__modified = False
 		logSys.debug(
 			(self.jail is not None and self.jail.name or "jailless") +
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 100ad233..7bd638d9 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -108,7 +108,6 @@ class FilterPyinotify(FileFilter):
 				self.jail.putFailTicket(ticket)
 		except FailManagerEmpty:
 			self.failManager.cleanup(MyTime.time())
-		self.dateDetector.sortTemplate()
 		self.__modified = False
 
 	def _addFileWatcher(self, path):
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 9a7d3e78..ab7310ba 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -143,13 +143,6 @@ class DateDetectorTest(LogCaptureTestCase):
 				else:
 					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
 
-	def testStableSortTemplate(self):
-		old_names = [x.name for x in self.__datedetector.templates]
-		self.__datedetector.sortTemplate()
-		# If there were no hits -- sorting should not change the order
-		for old_name, n in zip(old_names, self.__datedetector.templates):
-			self.assertEqual(old_name, n.name) # "Sort must be stable"
-
 	def testAllUniqueTemplateNames(self):
 		self.assertRaises(ValueError, self.__datedetector.appendTemplate,
 						  self.__datedetector.templates[0])
@@ -164,13 +157,11 @@ class DateDetectorTest(LogCaptureTestCase):
 		( logTime, logMatch ) = logdate
 		self.assertEqual(logTime, mu)
 		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
-		self.__datedetector.sortTemplate()
 		# confuse it with year being at the end
 		for i in xrange(10):
 			( logTime, logMatch ) =	self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
 			self.assertEqual(logTime, mu)
 			self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
-		self.__datedetector.sortTemplate()
 		# and now back to the original
 		( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
 		self.assertEqual(logTime, mu)
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 5edfd267..ae22b88f 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -87,8 +87,31 @@ def _maxWaitTime(wtime):
 	return wtime
 
 
-def _tm(time):
-	return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S")
+class _tmSerial():
+	_last_s = -0x7fffffff
+	_last_m = -0x7fffffff
+	_str_s = ""
+	_str_m = ""
+	@staticmethod
+	def _tm(time):
+		# ## strftime it too slow for large time serializer :
+		# return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S")
+		c = _tmSerial
+		sec = (time % 60)
+		if c._last_s == time - sec:
+			return "%s%02u" % (c._str_s, sec)
+		mt = (time % 3600)
+		if c._last_m == time - mt:
+			c._last_s = time - sec
+			c._str_s = "%s%02u:" % (c._str_m, mt // 60)
+			return "%s%02u" % (c._str_s, sec)
+		c._last_m = time - mt
+		c._str_m = datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:")
+		c._last_s = time - sec
+		c._str_s = "%s%02u:" % (c._str_m, mt // 60)
+		return "%s%02u" % (c._str_s, sec)
+
+_tm = _tmSerial._tm
 
 
 def _assert_equal_entries(utest, found, output, count=None):
@@ -245,6 +268,14 @@ class BasicFilter(unittest.TestCase):
 			1)
 		)
 
+	def testTest_tm(self):
+		unittest.F2B.SkipIfFast()
+		## test function "_tm" works correct (returns the same as slow strftime):
+		for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600):
+			tm = datetime.datetime.fromtimestamp(i).strftime("%Y-%m-%d %H:%M:%S")
+			if _tm(i) != tm:
+				self.assertEqual((_tm(i), i), (tm, i))
+
 
 class IgnoreIP(LogCaptureTestCase):
 

From da51fbf9c626a206c67e6b9e7dec61eadec2f8ba Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 29 Dec 2015 12:40:26 +0100
Subject: [PATCH 025/623] code review and performance optimization: - default
 date templates cache + regexp compiling on the fly (if required); - better
 usage of shared config over all test cases; - obsolete test cases removed
 (simple test of memleak/gc); - skip some slow test cases in fast mode (setup
 test, etc.);

---
 fail2ban/server/datedetector.py        | 133 +++++++++++++++----------
 fail2ban/server/datetemplate.py        |  12 +--
 fail2ban/tests/clientreadertestcase.py |  44 ++++----
 fail2ban/tests/failmanagertestcase.py  |  51 ----------
 fail2ban/tests/misctestcase.py         |   1 +
 fail2ban/tests/samplestestcase.py      |   3 +-
 fail2ban/tests/servertestcase.py       |  11 +-
 fail2ban/tests/utils.py                |   1 +
 8 files changed, 119 insertions(+), 137 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 4004d5c5..ae6d71d9 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -34,6 +34,82 @@ logSys = getLogger(__name__)
 logLevel = 6
 
 
+class DateDetectorCache(object):
+	def __init__(self):
+		self.__lock = Lock()
+		self.__templates = list()
+
+	@property
+	def templates(self):
+		"""List of template instances managed by the detector.
+		"""
+		with self.__lock:
+			if self.__templates:
+				return self.__templates
+			self._addDefaultTemplate()
+			return self.__templates
+
+	def _cacheTemplate(self, template):
+		"""Cache Fail2Ban's default template.
+		"""
+		if isinstance(template, str):
+			template = DatePatternRegex(template)
+		self.__templates.append(template)
+
+	def _addDefaultTemplate(self):
+		"""Add resp. cache Fail2Ban's default set of date templates.
+		"""
+		# asctime with optional day, subsecond and/or year:
+		# Sun Jan 23 21:59:59.011 2005 
+		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
+		# asctime with optional day, subsecond and/or year coming after day
+		# http://bugs.debian.org/798923
+		# Sun Jan 23 2005 21:59:59.011
+		self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?")
+		# simple date, optional subsecond (proftpd):
+		# 2005-01-23 21:59:59 
+		# simple date: 2005/01/23 21:59:59 
+		# custom for syslog-ng 2006.12.21 06:43:20
+		self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?")
+		# simple date too (from x11vnc): 23/01/2005 21:59:59 
+		# and with optional year given by 2 digits: 23/01/05 21:59:59 
+		# (See http://bugs.debian.org/537610)
+		# 17-07-2008 17:23:25
+		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S")
+		# Apache format optional time zone:
+		# [31/Oct/2006:09:22:55 -0000]
+		# 26-Jul-2007 15:20:52
+		self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
+		# CPanel 05/20/2008:01:57:39
+		self._cacheTemplate("%m/%d/%Y:%H:%M:%S")
+		# named 26-Jul-2007 15:20:52.252 
+		# roundcube 26-Jul-2007 15:20:52 +0200
+		# 01-27-2012 16:22:44.252
+		# subseconds explicit to avoid possible %m<->%d confusion
+		# with previous
+		self._cacheTemplate("%m-%d-%Y %H:%M:%S\.%f")
+		# TAI64N
+		template = DateTai64n()
+		template.name = "TAI64N"
+		self._cacheTemplate(template)
+		# Epoch
+		template = DateEpoch()
+		template.name = "Epoch"
+		self._cacheTemplate(template)
+		# ISO 8601
+		self._cacheTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?")
+		# Only time information in the log
+		self._cacheTemplate("^%H:%M:%S")
+		# <09/16/08@05:03:30>
+		self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>")
+		# MySQL: 130322 11:46:11
+		self._cacheTemplate("^%y%m%d  ?%H:%M:%S")
+		# Apache Tomcat
+		self._cacheTemplate("%b %d, %Y %I:%M:%S %p")
+		# ASSP: Apr-27-13 02:33:06
+		self._cacheTemplate("^%b-%d-%y %H:%M:%S")
+
+
 class DateDetector(object):
 	"""Manages one or more date templates to find a date within a log line.
 
@@ -41,6 +117,7 @@ class DateDetector(object):
 	----------
 	templates
 	"""
+	_defCache = DateDetectorCache()
 
 	def __init__(self):
 		self.__lock = Lock()
@@ -79,59 +156,9 @@ class DateDetector(object):
 	def addDefaultTemplate(self):
 		"""Add Fail2Ban's default set of date templates.
 		"""
-		self.__lock.acquire()
-		try:
-			# asctime with optional day, subsecond and/or year:
-			# Sun Jan 23 21:59:59.011 2005 
-			self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
-			# asctime with optional day, subsecond and/or year coming after day
-			# http://bugs.debian.org/798923
-			# Sun Jan 23 2005 21:59:59.011
-			self.appendTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?")
-			# simple date, optional subsecond (proftpd):
-			# 2005-01-23 21:59:59 
-			# simple date: 2005/01/23 21:59:59 
-			# custom for syslog-ng 2006.12.21 06:43:20
-			self.appendTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?")
-			# simple date too (from x11vnc): 23/01/2005 21:59:59 
-			# and with optional year given by 2 digits: 23/01/05 21:59:59 
-			# (See http://bugs.debian.org/537610)
-			# 17-07-2008 17:23:25
-			self.appendTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S")
-			# Apache format optional time zone:
-			# [31/Oct/2006:09:22:55 -0000]
-			# 26-Jul-2007 15:20:52
-			self.appendTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
-			# CPanel 05/20/2008:01:57:39
-			self.appendTemplate("%m/%d/%Y:%H:%M:%S")
-			# named 26-Jul-2007 15:20:52.252 
-			# roundcube 26-Jul-2007 15:20:52 +0200
-			# 01-27-2012 16:22:44.252
-			# subseconds explicit to avoid possible %m<->%d confusion
-			# with previous
-			self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f")
-			# TAI64N
-			template = DateTai64n()
-			template.name = "TAI64N"
-			self.appendTemplate(template)
-			# Epoch
-			template = DateEpoch()
-			template.name = "Epoch"
-			self.appendTemplate(template)
-			# ISO 8601
-			self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?")
-			# Only time information in the log
-			self.appendTemplate("^%H:%M:%S")
-			# <09/16/08@05:03:30>
-			self.appendTemplate("^<%m/%d/%y@%H:%M:%S>")
-			# MySQL: 130322 11:46:11
-			self.appendTemplate("^%y%m%d  ?%H:%M:%S")
-			# Apache Tomcat
-			self.appendTemplate("%b %d, %Y %I:%M:%S %p")
-			# ASSP: Apr-27-13 02:33:06
-			self.appendTemplate("^%b-%d-%y %H:%M:%S")
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			for template in DateDetector._defCache.templates:
+				self._appendTemplate(template)
 
 	@property
 	def templates(self):
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 88e2ae3f..746380ed 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -86,7 +86,6 @@ class DateTemplate(object):
 		if (wordBegin and not re.search(r'^\^', regex)):
 			regex = r'\b' + regex
 		self._regex = regex
-		self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE)
 
 	regex = property(getRegex, setRegex, doc=
 		"""Regex used to search for date.
@@ -95,6 +94,8 @@ class DateTemplate(object):
 	def matchDate(self, line):
 		"""Check if regex for date matches on a log line.
 		"""
+		if not self._cRegex:
+			self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE)
 		dateMatch = self._cRegex.search(line)
 		return dateMatch
 
@@ -171,7 +172,7 @@ class DatePatternRegex(DateTemplate):
 	regex
 	pattern
 	"""
-	_patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys())
+	_patternRE = re.compile(r"%%(%%|[%s])" % "".join(timeRE.keys()))
 	_patternName = {
 		'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
 		'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
@@ -202,10 +203,9 @@ class DatePatternRegex(DateTemplate):
 	@pattern.setter
 	def pattern(self, pattern):
 		self._pattern = pattern
-		self._name = re.sub(
-			self._patternRE, r'%(\1)s', pattern) % self._patternName
-		super(DatePatternRegex, self).setRegex(
-			re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE)
+		fmt = self._patternRE.sub(r'%(\1)s', pattern)
+		self._name = fmt % self._patternName
+		super(DatePatternRegex, self).setRegex(fmt % timeRE)
 
 	def setRegex(self, value):
 		raise NotImplementedError("Regex derived from pattern")
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index d0c37a3e..78c9a582 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -38,12 +38,15 @@ from ..client.configurator import Configurator
 from .utils import LogCaptureTestCase
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
+TEST_FILES_DIR_SHARE_CFG = {}
 
 from .utils import CONFIG_DIR
+CONFIG_DIR_SHARE_CFG = unittest.F2B.share_config
 
 STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
 
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
+IMPERFECT_CONFIG_SHARE_CFG = {}
 
 
 class ConfigReaderTest(unittest.TestCase):
@@ -162,14 +165,13 @@ class JailReaderTest(LogCaptureTestCase):
 
 	def __init__(self, *args, **kwargs):
 		super(JailReaderTest, self).__init__(*args, **kwargs)
-		self.__share_cfg = {}
 
 	def testIncorrectJail(self):
-		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg)
+		jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG)
 		self.assertRaises(ValueError, jail.read)
 		
 	def testJailActionEmpty(self):
-		jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
+		jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jail.read())
 		self.assertTrue(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
@@ -177,7 +179,7 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertLogged('No actions were defined for emptyaction')
 
 	def testJailActionFilterMissing(self):
-		jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
+		jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jail.read())
 		self.assertFalse(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
@@ -186,7 +188,7 @@ class JailReaderTest(LogCaptureTestCase):
 
 	def testJailActionBrokenDef(self):
 		jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG,
-			share_config=self.__share_cfg)
+			share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jail.read())
 		self.assertFalse(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
@@ -200,7 +202,7 @@ class JailReaderTest(LogCaptureTestCase):
 
 	if STOCK:
 		def testStockSSHJail(self):
-			jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
 			self.assertTrue(jail.read())
 			self.assertTrue(jail.getOptions())
 			self.assertFalse(jail.isEnabled())
@@ -316,7 +318,7 @@ class FilterReaderTest(unittest.TestCase):
 		self.assertEqual(sorted(filterReader.convert()), sorted(output))
 
 		filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"},
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
@@ -326,7 +328,7 @@ class FilterReaderTest(unittest.TestCase):
 	def testFilterReaderSubstitionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {},
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
@@ -335,7 +337,7 @@ class FilterReaderTest(unittest.TestCase):
 	def testFilterReaderSubstitionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
 		filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'},
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
@@ -346,7 +348,7 @@ class FilterReaderTest(unittest.TestCase):
 		filterName, filterOpt = JailReader.extractOptions(
 			'substition[honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
 		filterReader = FilterReader('substition', "jailname", filterOpt,
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
@@ -355,13 +357,13 @@ class FilterReaderTest(unittest.TestCase):
 	def testFilterReaderSubstitionFail(self):
 		# directly subst the same var :
 		filterReader = FilterReader('substition', "jailname", {'honeypot': '<honeypot>'},
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		self.assertRaises(ValueError, FilterReader.convert, filterReader)
 		# cross subst the same var :
 		filterReader = FilterReader('substition', "jailname", {'honeypot': '<sweet>', 'sweet': '<honeypot>'},
-		  share_config=self.__share_cfg, basedir=TEST_FILES_DIR)
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
 		filterReader.getOptions(None)
 		self.assertRaises(ValueError, FilterReader.convert, filterReader)
@@ -404,6 +406,7 @@ class JailsReaderTestCache(LogCaptureTestCase):
 		return cnt
 
 	def testTestJailConfCache(self):
+		unittest.F2B.SkipIfFast()
 		saved_ll = configparserinc.logLevel
 		configparserinc.logLevel = logging.DEBUG
 		basedir = tempfile.mkdtemp("fail2ban_conf")
@@ -446,7 +449,6 @@ class JailsReaderTest(LogCaptureTestCase):
 
 	def __init__(self, *args, **kwargs):
 		super(JailsReaderTest, self).__init__(*args, **kwargs)
-		self.__share_cfg = {}
 
 	def testProvidingBadBasedir(self):
 		if not os.path.exists('/XXX'):
@@ -454,7 +456,7 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertRaises(ValueError, reader.read)
 
 	def testReadTestJailConf(self):
-		jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
+		jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jails.read())
 		self.assertFalse(jails.getOptions())
 		self.assertRaises(ValueError, jails.convert)
@@ -504,7 +506,7 @@ class JailsReaderTest(LogCaptureTestCase):
 						msg="Action file %r is lacking [Init] section" % actionConfig)
 
 		def testReadStockJailConf(self):
-			jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
 			comm_commands = jails.convert()
@@ -535,7 +537,7 @@ class JailsReaderTest(LogCaptureTestCase):
 				# moreover we must have a file for it
 				# and it must be readable as a Filter
 				filterReader = FilterReader(filterName, jail, filterOpt, 
-					share_config=self.__share_cfg, basedir=CONFIG_DIR)
+					share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
 				self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName)		  # opens fine
 				filterReader.getOptions({})	  # reads fine
 
@@ -554,8 +556,8 @@ class JailsReaderTest(LogCaptureTestCase):
 					if actName == 'iptables-multiport':
 						self.assertTrue('port' in actOpt)
 
-					actionReader = ActionReader(
-						actName, jail, {}, basedir=CONFIG_DIR)
+					actionReader = ActionReader(actName, jail, {}, 
+						share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
 					self.assertTrue(actionReader.read())
 					actionReader.getOptions({})	  # populate _opts
 					cmds = actionReader.convert()
@@ -566,7 +568,7 @@ class JailsReaderTest(LogCaptureTestCase):
 
 		# Verify that all filters found under config/ have a jail
 		def testReadStockJailFilterComplete(self):
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg)
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG)
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
@@ -586,7 +588,7 @@ class JailsReaderTest(LogCaptureTestCase):
 		def testReadStockJailConfForceEnabled(self):
 			# more of a smoke test to make sure that no obvious surprises
 			# on users' systems when enabling shipped jails
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
 			comm_commands = jails.convert(allow_no_files=True)
@@ -697,7 +699,7 @@ action = testaction1[actname=test1]
 filter = testfilter1
 """)
 		jailfd.close()
-		jails = JailsReader(basedir=basedir, share_config=self.__share_cfg)
+		jails = JailsReader(basedir=basedir, share_config={})
 		self.assertTrue(jails.read())
 		self.assertTrue(jails.getOptions())
 		comm_commands = jails.convert(allow_no_files=True)
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 3cac0540..a8a71723 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -248,55 +248,4 @@ class FailmanagerComplex(unittest.TestCase):
 				self.assertEqual(str(ip), '127.0.255.255')
 			elif i == 65535:
 				self.assertEqual(str(ip), '127.1.0.0')
-	
-	def testFailuresMemLeak1(self):
-		# use factor (divisor) instead unittest.F2B.SkipIfFast() :
-		modeDiv = 1 
-		if unittest.F2B.fast: # pragma: no cover
-			modeDiv = 10
-			self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv)
-		import gc
-		gc.collect()
-		timestamp = 1167606999.0
-		ticktime = timestamp-(1000 // modeDiv)
-		for i, ip in self._ip_range(30000 // modeDiv):
-			t = FailTicket(ip, ticktime, list({'match': i}))
-			ticktime += 1
-			self.__failManager.addFailure(t)
-			if i % (500 // modeDiv) == 0:
-			 	self.__failManager.cleanup(timestamp)
-			if i % (888 // modeDiv) == 0:
-				timestamp += (1000 // modeDiv)
-				ticktime = timestamp-(1000 // modeDiv)
-		self.assertFalse(gc.collect())
-		self.__failManager.cleanup(timestamp)
-		ticktime -= timestamp - self.__failManager.getMaxTime() + 1
-		self.assertEqual(self.__failManager.size(), ticktime if ticktime > 0 else 0)
-		self.assertFalse(gc.collect())
-
-	def testFailuresMemLeak2(self):
-		# use factor (divisor) instead unittest.F2B.SkipIfFast() :
-		modeDiv = 1 
-		if unittest.F2B.fast: # pragma: no cover
-			modeDiv = 10
-			self.__failManager.setMaxTime(self.__failManager.getMaxTime() // modeDiv)
-		import gc
-		gc.collect()
-		timestamp = 1167606999.0
-		ticktime = timestamp-(1000 // modeDiv)
-		for i, ip in self._ip_range(10000 // modeDiv):
-			for j in range(0, 5):
-				t = FailTicket(ip, ticktime, list({'match': i}))
-				ticktime += 1
-				self.__failManager.addFailure(t)
-			if i % (500 // modeDiv) == 0:
-			 	self.__failManager.cleanup(timestamp)
-			if i % (888 // modeDiv) == 0 or i % (1500 // modeDiv) == 0:
-				timestamp += (1000 // modeDiv)
-				ticktime = timestamp-(1000 // modeDiv)
-		self.assertFalse(gc.collect())
-		timestamp += 20000
-		self.__failManager.cleanup(timestamp)
-		self.assertEqual(self.__failManager.size(), 0)
-		self.assertFalse(gc.collect())
 
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index c4dd716d..c9eddca1 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -80,6 +80,7 @@ def _getSysPythonVersion():
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
+		unittest.F2B.SkipIfFast()
 		setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py')
 		self.setup = os.path.exists(setup) and setup or None
 		if not self.setup and sys.version_info >= (2,7): # pragma: no cover - running not out of the source
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 9e6c0ee7..2ed77554 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -64,7 +64,8 @@ def testSampleRegexsFactory(name):
 	def testFilter(self):
 
 		# Check filter exists
-		filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR)
+		filterConf = FilterReader(name, "jail", {}, 
+			basedir=CONFIG_DIR, share_config=unittest.F2B.share_config)
 		self.assertEqual(filterConf.getFile(), name)
 		self.assertEqual(filterConf.getJailName(), "jail")
 		filterConf.read()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 87ca4289..96734262 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -47,6 +47,7 @@ except ImportError: # pragma: no cover
 	filtersystemd = None
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
+FAST_BACKEND = "polling"
 
 
 class TestServer(Server):
@@ -72,7 +73,7 @@ class TransmitterBase(unittest.TestCase):
 		self.tmp_files.append(pidfile_name)
 		self.server.start(sock_name, pidfile_name, force=False)
 		self.jailName = "TestJail1"
-		self.server.addJail(self.jailName, "auto")
+		self.server.addJail(self.jailName, FAST_BACKEND)
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -195,7 +196,7 @@ class Transmitter(TransmitterBase):
 		self.setGetTest("dbpurgeage", "600", 600)
 		self.setGetTestNOK("dbpurgeage", "LIZARD")
 		# the same file name (again with jails / not changed):
-		self.server.addJail(self.jailName, "auto")
+		self.server.addJail(self.jailName, FAST_BACKEND)
 		self.setGetTest("dbfile", tmpFilename)
 		self.server.delJail(self.jailName)
 
@@ -213,7 +214,7 @@ class Transmitter(TransmitterBase):
 			["get", "dbpurgeage"]),
 			(0, None))
 		# the same (again with jails / not changed):
-		self.server.addJail(self.jailName, "auto")
+		self.server.addJail(self.jailName, FAST_BACKEND)
 		self.assertEqual(self.transm.proceed(
 			["set", "dbfile", "None"]),
 			(0, None))
@@ -252,7 +253,7 @@ class Transmitter(TransmitterBase):
 		self.assertTrue(self.jailName not in self.server._Server__jails)
 
 	def testStartStopAllJail(self):
-		self.server.addJail("TestJail2", "auto")
+		self.server.addJail("TestJail2", FAST_BACKEND)
 		self.assertEqual(
 			self.transm.proceed(["start", self.jailName]), (0, None))
 		self.assertEqual(
@@ -497,7 +498,7 @@ class Transmitter(TransmitterBase):
 		jails = [self.jailName]
 		self.assertEqual(self.transm.proceed(["status"]),
 			(0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))]))
-		self.server.addJail("TestJail2", "auto")
+		self.server.addJail("TestJail2", FAST_BACKEND)
 		jails.append("TestJail2")
 		self.assertEqual(self.transm.proceed(["status"]),
 			(0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))]))
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index c33c2b19..0cac9df5 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -59,6 +59,7 @@ class F2B(optparse.Values):
 		if self.fast:
 			self.memory_db = True
 			self.no_gamin = True
+		self.__dict__['share_config'] = {}
 	def SkipIfFast(self):
 		pass
 	def SkipIfNoNetwork(self):

From 8eca2ecd84182242973aed9fba5760bd32947dc9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 29 Dec 2015 17:38:47 +0100
Subject: [PATCH 026/623] optimized version of "str2seconds" with pre-compiled
 regexp;

---
 fail2ban/server/mytime.py | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py
index 329e2465..816b04c4 100644
--- a/fail2ban/server/mytime.py
+++ b/fail2ban/server/mytime.py
@@ -98,6 +98,25 @@ class MyTime:
 		else:
 			return time.localtime(MyTime.myTime)
 
+	## precreate/precompile primitives used in str2seconds:
+
+	## preparing expression:
+	_str2sec_prep = re.compile(r"(?i)(?<=[a-z])(\d)")
+	## finally expression:
+	_str2sec_fini = re.compile(r"(\d)\s+(\d)")
+	## wrapper for each sub part:
+	_str2sec_subpart = r"(?i)(?<=[\d\s])(%s)\b"
+	## parts to be replaced - pair of (regexp x replacement):
+	_str2sec_parts = (
+		(re.compile(_str2sec_subpart % r"days?|da|dd?"),      "*"+str(24*60*60)),
+		(re.compile(_str2sec_subpart % r"weeks?|wee?|ww?"),   "*"+str(7*24*60*60)),
+		(re.compile(_str2sec_subpart % r"months?|mon?"),      "*"+str((365*3+366)*24*60*60/4/12)),
+		(re.compile(_str2sec_subpart % r"years?|yea?|yy?"),   "*"+str((365*3+366)*24*60*60/4)),
+		(re.compile(_str2sec_subpart % r"seconds?|sec?|ss?"), "*"+str(1)),
+		(re.compile(_str2sec_subpart % r"minutes?|min?|mm?"), "*"+str(60)),
+		(re.compile(_str2sec_subpart % r"hours?|hou?|hh?"),   "*"+str(60*60)),
+	)
+
 	@staticmethod
 	def str2seconds(val):
 		"""Wraps string expression like "1h 2m 3s" into number contains seconds (3723).
@@ -120,13 +139,9 @@ class MyTime:
 		if isinstance(val, (int, long, float, complex)):
 			return val
 		# replace together standing abbreviations, example '1d12h' -> '1d 12h':
-		val = re.sub(r"(?i)(?<=[a-z])(\d)", r" \1", val)
+		val = MyTime._str2sec_prep.sub(r" \1", val)
 		# replace abbreviation with expression:
-		for rexp, rpl in (
-			(r"days?|da|dd?", 24*60*60), (r"weeks?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), 
-			(r"years?|yea?|yy?", (365*3+366)*24*60*60/4), 
-			(r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|hou?|hh?", 60*60),
-		):
-			val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val)
-		val = re.sub(r"(\d)\s+(\d)", r"\1+\2", val);
+		for rexp, rpl in MyTime._str2sec_parts:
+			val = rexp.sub(rpl, val)
+		val = MyTime._str2sec_fini.sub(r"\1+\2", val)
 		return eval(val)

From 134c33cc6d11249c5f1532f098345681e1875eec Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 29 Dec 2015 17:40:37 +0100
Subject: [PATCH 027/623] fail2ban-testcases: added option "--ignore": negate
 [regexps] filter, to ignore tests matched specified regexps

---
 bin/fail2ban-testcases  | 3 +++
 fail2ban/tests/utils.py | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 22a44a8b..606b0b06 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -67,6 +67,9 @@ def get_opt_parser():
 		Option('-f', "--fast", action="store_true",
 			   dest="fast",
 			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
+		Option('-i', "--ignore", action="store_true",
+			   dest="negate_re",
+			   help="negate [regexps] filter to ignore tests matched specified regexps"),
 		Option("-t", "--log-traceback", action='store_true',
 			   help="Enrich log-messages with compressed tracebacks"),
 		Option("--full-traceback", action='store_true',
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 0cac9df5..ce6d638a 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -55,6 +55,7 @@ class F2B(optparse.Values):
 	def __init__(self, opts={}):
 		self.__dict__ = opts.__dict__ if opts else {
 			'fast': False, 'memory_db':False, 'no_gamin': False, 'no_network': False, 
+			"negate_re": False,
 		}
 		if self.fast:
 			self.memory_db = True
@@ -157,7 +158,8 @@ def gatherTests(regexps=None, opts=None):
 				for test in suite:
 					s = str(test)
 					for r in self._regexps:
-						if r.search(s):
+						m = r.search(s)
+						if (m if not opts.negate_re else not m):
 							matched.append(test)
 							break
 				for test in matched:

From dca3db941e049d79c6ffcbd26a235a512d29eea1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 7 Mar 2016 17:28:37 +0100
Subject: [PATCH 028/623] merge getTime2 functionality to getTime

---
 fail2ban/server/datedetector.py        | 49 +++++++-------------------
 fail2ban/server/filter.py              |  4 +--
 fail2ban/tests/datedetectortestcase.py |  4 +--
 3 files changed, 17 insertions(+), 40 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index ae6d71d9..0c5da94b 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -202,13 +202,13 @@ class DateDetector(object):
 		# not found:
 		return (None, None)
 
-	def getTime(self, line):
+	def getTime(self, line, timeMatch=None):
 		"""Attempts to return the date on a log line using templates.
 
-		Obsolete: Use "getTime2" instead.
-
 		This uses the templates' `getDate` method in an attempt to find
-		a date.
+		a date. 
+		For the faster usage, always specify a parameter timeMatch (the previous tuple result
+		of the matchTime), then this will work without locking and without cycle over templates.
 
 		Parameters
 		----------
@@ -221,6 +221,15 @@ class DateDetector(object):
 			The Unix timestamp returned from the first successfully matched
 			template or None if not found.
 		"""
+		if timeMatch:
+			template = timeMatch[1]
+			if template is not None:
+				date = template.getDate(line, timeMatch[0])
+				if date is not None:
+					if logSys.getEffectiveLevel() <= logLevel:
+						logSys.log(logLevel, "Got time %f for \"%r\" using template %s",
+							date[0], date[1].group(), template.name)
+					return date
 		with self.__lock:
 			for template in self.__templates:
 				try:
@@ -235,38 +244,6 @@ class DateDetector(object):
 					pass
 			return None
 
-	def getTime2(self, line, timeMatch = None):
-		"""Attempts to return the date on a log line using given template.
-
-		This uses the templates' `getDate` method in an attempt to find
-		a date.
-		Method 'getTime2' is a little bit faster as 'getTime' if template was specified (cause works without locking and without cycle)
-
-		Parameters
-		----------
-		line : str
-			Line which is searched by the date templates.
-		timeMatch (timeMatch, template) : (Match, DateTemplate)
-			Time match and template previously returned from matchTime
-
-		Returns
-		-------
-		float
-			The Unix timestamp returned from the first successfully matched
-			template or None if not found.
-		"""
-		date = None
-		if timeMatch:
-			template = timeMatch[1]
-			if template is not None:
-				date = template.getDate(line, timeMatch[0])
-		if date is not None:
-			if logSys.getEffectiveLevel() <= logLevel:
-				logSys.log(logLevel, "Got time(2) %f for \"%r\" using template %s",
-					date[0], date[1].group(), template.name)
-			return date
-		return self.getTime(line)
-
 	def _reorderTemplate(self, num):
 		"""Reorder template (bubble up) in template list if hits grows enough.
 
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 8dbb6a7a..b7de21c8 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -479,7 +479,7 @@ class Filter(JailThread):
 			self.__lastDate = date
 		elif timeText:
 
-			dateTimeMatch = self.dateDetector.getTime2(timeText, tupleLine[3])
+			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
 
 			if dateTimeMatch is None:
 				logSys.error("findFailure failed to parse timeText: " + timeText)
@@ -769,7 +769,7 @@ class FileFilter(Filter):
 					break
 				(timeMatch, template) = self.dateDetector.matchTime(line)
 				if timeMatch:
-					dateTimeMatch = self.dateDetector.getTime2(line[timeMatch.start():timeMatch.end()], (timeMatch, template))
+					dateTimeMatch = self.dateDetector.getTime(line[timeMatch.start():timeMatch.end()], (timeMatch, template))
 				else:
 					nextp = container.tell()
 					if nextp > maxp:
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index ab7310ba..dc01fd49 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -128,9 +128,9 @@ class DateDetectorTest(LogCaptureTestCase):
 						self.assertEqual(logMatch.group(), sdate)
 				else:
 					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
-				# with matchTime and getTime2 (this combination used in filter) :
+				# with getTime(matchTime) - this combination used in filter:
 				matchTime = self.__datedetector.matchTime(log)
-				logtime = self.__datedetector.getTime2(log, matchTime)
+				logtime = self.__datedetector.getTime(log, matchTime)
 				if should_match:
 					self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log))
 					( logUnix, logMatch ) = logtime

From 7eafb0c2066dd147c054bee7dfa34eac4075a3ea Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 9 Mar 2016 22:02:07 -0500
Subject: [PATCH 029/623] ENH: avoid subclassing dict (in favor of delegating)
 if not entire interface is used/adjusted

since otherwise use of dict"s __getitem__ could lead to undesired effects etc
---
 fail2ban/server/utils.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 45d1c09d..ad2607c5 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -50,34 +50,41 @@ class Utils():
 	DEFAULT_SLEEP_INTERVAL = 0.01
 
 
-	class Cache(dict):
+	class Cache(object):
+		"""A simple cache with a TTL and limit on size
+		"""
 
 		def __init__(self, *args, **kwargs):
 			self.setOptions(*args, **kwargs)
+			self._cache = {}
 
 		def setOptions(self, maxCount=1000, maxTime=60):
 			self.maxCount = maxCount
 			self.maxTime = maxTime
 
+		def __len__(self):
+			return len(self._cache)
+
 		def get(self, k, defv=None):
-			v = dict.get(self, k)
+			v = self._cache.get(k)
 			if v: 
 				if v[1] > time.time():
 					return v[0]
-				del self[k]
+				del self._cache[k]
 			return defv
 			
 		def set(self, k, v):
 			t = time.time()
+			cache = self._cache  # for shorter local access
 			# clean cache if max count reached:
-			if len(self) >= self.maxCount:
-				for (ck,cv) in self.items():
+			if len(cache) >= self.maxCount:
+				for (ck, cv) in cache.items():
 					if cv[1] < t:
-						del self[ck]
+						del cache[ck]
 				# if still max count - remove any one:
-				if len(self) >= self.maxCount:
-					self.popitem()
-			self[k] = (v, t + self.maxTime)
+				if len(cache) >= self.maxCount:
+					cache.popitem()
+			cache[k] = (v, t + self.maxTime)
 
 
 	@staticmethod

From c84b6370c548884fd17028d4a5f3f86efd89171c Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 9 Mar 2016 22:02:51 -0500
Subject: [PATCH 030/623] ENH: remove some code duplication, enhance
 docstrings, uniformize naming

---
 fail2ban/server/filter.py |  6 +++---
 fail2ban/server/utils.py  | 14 ++++++++------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b0ccda84..f5cb8546 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -998,7 +998,7 @@ class DNSUtils:
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 
 	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_dnsToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
 	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
 
 	@staticmethod
@@ -1007,7 +1007,7 @@ class DNSUtils:
 			Thanks to Kevin Drapel.
 		"""
 		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		v = DNSUtils.CACHE_dnsToIp.get(dns)
+		v = DNSUtils.CACHE_nameToIp.get(dns)
 		if v is not None: 
 			return v
 		# retrieve ip (todo: use AF_INET6 for IPv6)
@@ -1017,7 +1017,7 @@ class DNSUtils:
 			# todo: make configurable the expired time of cache entry:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
 			v = list()
-		DNSUtils.CACHE_dnsToIp.set(dns, v)
+		DNSUtils.CACHE_nameToIp.set(dns, v)
 		return v
 
 	@staticmethod
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index ad2607c5..cd05bf93 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -111,12 +111,13 @@ class Utils():
 			If shell is True (default), the specified command (may be a string) will be 
 			executed through the shell.
 		output : bool
-			If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode)
+			If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode).
+			If False, just indication of success is returned
 
 		Returns
 		-------
-		bool
-			True if the command succeeded.
+		bool or (bool, str, str, int)
+			True if the command succeeded and with stdout, stderr, returncode if output was set to True
 
 		Raises
 		------
@@ -185,9 +186,10 @@ class Utils():
 					logSys.log(std_level, "%s -- stderr: %r", realCmd, stderr)
 				popen.stderr.close()
 
+		success = False
 		if retcode == 0:
 			logSys.debug("%s -- returned successfully", realCmd)
-			return True if not output else (True, stdout, stderr, retcode)
+			success = True
 		elif retcode is None:
 			logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid))
 		elif retcode < 0 or retcode > 128:
@@ -200,13 +202,13 @@ class Utils():
 			logSys.error("%s -- returned %i" % (realCmd, retcode))
 			if msg:
 				logSys.info("HINT on %i: %s", retcode, msg % locals())
-		return False if not output else (False, stdout, stderr, retcode)
+		return success if not output else (success, stdout, stderr, retcode)
 	
 	@staticmethod
 	def wait_for(cond, timeout, interval=None):
 		"""Wait until condition expression `cond` is True, up to `timeout` sec
 		"""
-		ini = 1
+		ini = 1  # to delay initializations until/when necessary
 		while True:
 			ret = cond()
 			if ret:

From deca0b80abcbdd6129d7448d8499fe31cb4ec49e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 9 Mar 2016 22:17:14 -0500
Subject: [PATCH 031/623] ENH: removed double tab indentation, pass use_poll
 into loop, reorderd log msg to come after action to be factually correct

---
 fail2ban/server/asyncserver.py | 48 ++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index ad37544a..d43874c7 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -92,26 +92,29 @@ class RequestHandler(asynchat.async_chat):
 		
 
 def loop(active, timeout=None, use_poll=False):
-		# Use poll instead of loop, because of recognition of active flag, 
-		# because of loop timeout mistake: different in poll and poll2 (sec vs ms),
-		# and to prevent sporadical errors like EBADF 'Bad file descriptor' etc. (see gh-161)
-		if timeout is None:
-			timeout = Utils.DEFAULT_SLEEP_TIME
-		poll = asyncore.poll
-		if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover
-			logSys.debug('Server listener (select) uses poll')
-			# poll2 expected a timeout in milliseconds (but poll and loop in seconds):
-			timeout = float(timeout) / 1000
-			poll = asyncore.poll2
-		# Poll as long as active:
-		while active():
-			try:
-				poll(timeout)
-			except Exception as e: # pragma: no cover
-				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor')
-					logSys.info('Server connection was closed: %s', str(e))
-				else:
-					logSys.error('Server connection was closed: %s', str(e))
+	"""Custom event loop implementation
+
+	Uses poll instead of loop to respect `active` flag,
+	to avoid loop timeout mistake: different in poll and poll2 (sec vs ms),
+	and to prevent sporadic errors like EBADF 'Bad file descriptor' etc. (see gh-161)
+	"""
+	if timeout is None:
+		timeout = Utils.DEFAULT_SLEEP_TIME
+	poll = asyncore.poll
+	if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'):  # pragma: no cover
+		logSys.debug('Server listener (select) uses poll')
+		# poll2 expected a timeout in milliseconds (but poll and loop in seconds):
+		timeout = float(timeout) / 1000
+		poll = asyncore.poll2
+	# Poll as long as active:
+	while active():
+		try:
+			poll(timeout)
+		except Exception as e:  # pragma: no cover
+			if e.args[0] in (errno.ENOTCONN, errno.EBADF):  # (errno.EBADF, 'Bad file descriptor')
+				logSys.info('Server connection was closed: %s', str(e))
+			else:
+				logSys.error('Server connection was closed: %s', str(e))
 
 
 ##
@@ -177,7 +180,7 @@ class AsyncServer(asyncore.dispatcher):
 		# Sets the init flag.
 		self.__init = self.__active = True
 		# Event loop as long as active:
-		loop(lambda: self.__active)
+		loop(lambda: self.__active, use_poll=use_poll)
 		# Cleanup all
 		self.stop()
 
@@ -187,8 +190,8 @@ class AsyncServer(asyncore.dispatcher):
 			asyncore.dispatcher.close(self)
 		# Remove socket (file) only if it was created:
 		if self.__init and os.path.exists(self.__sock):
-			logSys.debug("Removed socket file " + self.__sock)
 			os.remove(self.__sock)
+			logSys.debug("Removed socket file " + self.__sock)
 		logSys.debug("Socket shutdown")
 		self.__active = False
 
@@ -198,6 +201,7 @@ class AsyncServer(asyncore.dispatcher):
 	def stop(self):
 		self.close()
 
+	# better remains a method (not a property) since used as a callable for wait_for
 	def isActive(self):
 		return self.__active
 

From 01b379ab2ebb4b3b9d40e4e1cb0fc8b0cca49f18 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 10 Mar 2016 11:47:26 +0100
Subject: [PATCH 032/623] fix sporadic bugs within asyncserver, cherry picked
 from "f2b-perfom-prepare-716-cs": - differentiate between in loop and active
 (reset active only after really leaves a loop); - differentiate between
 foreign and self loop-thread by stop/close connection; - stops logging
 connection errors if too many errors; - safe remove socket (no sporadic
 errors if too fast start/stop occurred);

---
 fail2ban/server/asyncserver.py | 48 +++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index d43874c7..bb237cd7 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -32,6 +32,7 @@ import fcntl
 import os
 import socket
 import sys
+import threading
 import traceback
 
 from .utils import Utils
@@ -98,10 +99,11 @@ def loop(active, timeout=None, use_poll=False):
 	to avoid loop timeout mistake: different in poll and poll2 (sec vs ms),
 	and to prevent sporadic errors like EBADF 'Bad file descriptor' etc. (see gh-161)
 	"""
+	errCount = 0
 	if timeout is None:
 		timeout = Utils.DEFAULT_SLEEP_TIME
 	poll = asyncore.poll
-	if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'):  # pragma: no cover
+	if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover
 		logSys.debug('Server listener (select) uses poll')
 		# poll2 expected a timeout in milliseconds (but poll and loop in seconds):
 		timeout = float(timeout) / 1000
@@ -110,11 +112,20 @@ def loop(active, timeout=None, use_poll=False):
 	while active():
 		try:
 			poll(timeout)
-		except Exception as e:  # pragma: no cover
-			if e.args[0] in (errno.ENOTCONN, errno.EBADF):  # (errno.EBADF, 'Bad file descriptor')
-				logSys.info('Server connection was closed: %s', str(e))
-			else:
-				logSys.error('Server connection was closed: %s', str(e))
+			if errCount:
+				errCount -= 1
+		except Exception as e: # pragma: no cover
+			if not active():
+				break
+			errCount += 1
+			if errCount < 20:
+				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor')
+					logSys.info('Server connection was closed: %s', str(e))
+				else:
+					logSys.error('Server connection was closed: %s', str(e))
+			elif errCount == 20:
+				logSys.info('Too many errors - stop logging connection errors')
+				logSys.exception(e)
 
 
 ##
@@ -159,13 +170,14 @@ class AsyncServer(asyncore.dispatcher):
 	# @param force: remove the socket file if exists.
 	
 	def start(self, sock, force, use_poll=False):
+		self.__worker = threading.current_thread()
 		self.__sock = sock
 		# Remove socket
 		if os.path.exists(sock):
 			logSys.error("Fail2ban seems to be already running")
 			if force:
 				logSys.warning("Forcing execution of the server")
-				os.remove(sock)
+				self._remove_sock()
 			else:
 				raise AsyncServerException("Server already running")
 		# Creates the socket.
@@ -178,19 +190,25 @@ class AsyncServer(asyncore.dispatcher):
 		AsyncServer.__markCloseOnExec(self.socket)
 		self.listen(1)
 		# Sets the init flag.
-		self.__init = self.__active = True
+		self.__init = self.__loop = self.__active = True
 		# Event loop as long as active:
-		loop(lambda: self.__active, use_poll=use_poll)
+		loop(lambda: self.__loop, use_poll=use_poll)
+		self.__active = False
 		# Cleanup all
 		self.stop()
 
 
 	def close(self):
 		if self.__active:
+			self.__loop = False
 			asyncore.dispatcher.close(self)
+			# If not the loop thread (stops self in handler), wait (a little bit) 
+			# for the server leaves loop, before remove socket
+			if threading.current_thread() != self.__worker:
+				Utils.wait_for(lambda: not self.__active, 1)
 		# Remove socket (file) only if it was created:
 		if self.__init and os.path.exists(self.__sock):
-			os.remove(self.__sock)
+			self._remove_sock()
 			logSys.debug("Removed socket file " + self.__sock)
 		logSys.debug("Socket shutdown")
 		self.__active = False
@@ -205,6 +223,16 @@ class AsyncServer(asyncore.dispatcher):
 	def isActive(self):
 		return self.__active
 
+	##
+	# Safe remove (in multithreaded mode):
+
+	def _remove_sock(self):
+		try:
+			os.remove(self.__sock)
+		except OSError as e:
+			if e.errno != errno.ENOENT:
+				raise
+
 	##
 	# Marks socket as close-on-exec to avoid leaking file descriptors when
 	# running actions involving command execution.

From 74b88e1706ba40e735314defb9f71beb8cc700b7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 10 Mar 2016 12:05:59 +0100
Subject: [PATCH 033/623] fix sporadic bugs by starting of fast actions
 (execute fast sub processes), cherry picked from "f2b-perfom-prepare-716-cs":
 - missing `popen` variable if not started; - better readability for stdout
 and stderr in log (+ check log level before do logging);

---
 fail2ban/server/utils.py         | 18 +++++++++++++-----
 fail2ban/tests/actiontestcase.py |  4 ++--
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index cd05bf93..738394c3 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -133,6 +133,7 @@ class Utils():
 			timeout_expr = lambda: time.time() - stime <= timeout
 		else:
 			timeout_expr = timeout
+		popen = None
 		try:
 			popen = subprocess.Popen(
 				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell,
@@ -159,7 +160,10 @@ class Utils():
 				if retcode is None and not Utils.pid_exists(pgid):
 					retcode = signal.SIGKILL
 		except OSError as e:
-			logSys.error("%s -- failed with %s" % (realCmd, e))
+			stderr = "%s -- failed with %s" % (realCmd, e)
+			logSys.error(stderr)
+			if not popen:
+				return False if not output else (False, stdout, stderr, retcode)
 
 		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
 		# if we need output (to return or to log it): 
@@ -172,8 +176,10 @@ class Utils():
 					stdout = popen.stdout.read()
 				except IOError as e:
 					logSys.error(" ... -- failed to read stdout %s", e)
-				if stdout is not None and stdout != '':
-					logSys.log(std_level, "%s -- stdout: %r", realCmd, stdout)
+				if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
+					logSys.log(std_level, "%s -- stdout:", realCmd)
+					for l in stdout.splitlines():
+						logSys.log(std_level, " -- stdout: %r", l)
 				popen.stdout.close()
 			if popen.stderr:
 				try:
@@ -182,8 +188,10 @@ class Utils():
 					stderr = popen.stderr.read()
 				except IOError as e:
 					logSys.error(" ... -- failed to read stderr %s", e)
-				if stderr is not None and stderr != '':
-					logSys.log(std_level, "%s -- stderr: %r", realCmd, stderr)
+				if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
+					logSys.log(std_level, "%s -- stderr:", realCmd)
+					for l in stderr.splitlines():
+						logSys.log(std_level, " -- stderr: %r", l)
 				popen.stderr.close()
 
 		success = False
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 6d8fcc82..1872eb1f 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -271,11 +271,11 @@ class CommandActionTest(LogCaptureTestCase):
 
 	def testCaptureStdOutErr(self):
 		CommandAction.executeCmd('echo "How now brown cow"')
-		self.assertLogged("'How now brown cow\\n'")
+		self.assertLogged("stdout: 'How now brown cow'\n", "stdout: b'How now brown cow'\n")
 		CommandAction.executeCmd(
 			'echo "The rain in Spain stays mainly in the plain" 1>&2')
 		self.assertLogged(
-			"'The rain in Spain stays mainly in the plain\\n'")
+			"stderr: 'The rain in Spain stays mainly in the plain'\n", "stderr: b'The rain in Spain stays mainly in the plain'\n")
 
 	def testCallingMap(self):
 		mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'),

From 5773eb71b8cc738d1e2a024a93fce245818d0fcd Mon Sep 17 00:00:00 2001
From: Alexey Korepanov <kaikaikai@yandex.ru>
Date: Tue, 12 Apr 2016 21:20:19 +0100
Subject: [PATCH 034/623] ensure /var/run/fail2ban is created in systemd
 service file

---
 files/fail2ban.service | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/fail2ban.service b/files/fail2ban.service
index e79faed1..985f5c1f 100644
--- a/files/fail2ban.service
+++ b/files/fail2ban.service
@@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service
 
 [Service]
 Type=forking
+ExecStartPre=/bin/mkdir -p /var/run/fail2ban
 ExecStart=/usr/bin/fail2ban-client -x start
 ExecStop=/usr/bin/fail2ban-client stop
 ExecReload=/usr/bin/fail2ban-client reload

From 01e1383c9bb3b6f75f087d2d2303b37811fa5e32 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 28 Feb 2016 11:11:24 +0100
Subject: [PATCH 035/623] New class IPAddr for handling IPv4 and IPv6 addresses

---
 fail2ban/server/filter.py | 243 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 239 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index f5cb8546..da5fc1ea 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -433,7 +433,7 @@ class Filter(JailThread):
 				logSys.debug("Ignore line since time %s < %s - %s", 
 					unixTime, MyTime.time(), self.getFindTime())
 				break
-			if self.inIgnoreIPList(ip, log_ignore=True):
+			if self.inIgnoreIPList(ip.ntoa(), log_ignore=True):
 				continue
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
@@ -530,7 +530,8 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							failList.append([failRegexIndex, host, date,
+							ipaddr = IPAddr(host)
+							failList.append([failRegexIndex, ipaddr, date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
 								break
@@ -538,8 +539,9 @@ class Filter(JailThread):
 							ipMatch = DNSUtils.textToIp(host, self.__useDns)
 							if ipMatch:
 								for ip in ipMatch:
-									failList.append([failRegexIndex, ip, date,
-										 failRegex.getMatchedLines()])
+									ipaddr = IPAddr(ip)
+									failList.append([failRegexIndex, ipaddr, 
+										 date, failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
@@ -1096,3 +1098,236 @@ class DNSUtils:
 		""" Convert a binary IPv4 address into string n.n.n.n form.
 		"""
 		return socket.inet_ntoa(struct.pack("!L", ipbin))
+
+
+
+##
+# Class for IP address handling.
+#
+# This class contains methods for handling IPv4 and IPv6 addresses.
+
+class IPAddr:
+	""" provide functions to handle IPv4 and IPv6 addresses 
+	"""
+
+	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
+	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+
+	# object attributes
+	addr = 0
+	family = socket.AF_UNSPEC
+	plen = 0
+	valid = False
+	raw = ""
+
+	# object methods
+	def __init__(self, ipstring, cidr=-1):
+		""" initialize IP object by converting IP address string
+			to binary to integer
+		"""
+		for family in [socket.AF_INET, socket.AF_INET6]:
+			try:
+				binary = socket.inet_pton(family, ipstring)
+			except socket.error:
+				continue
+			else: 
+				self.valid = True
+				break
+
+		if self.valid and family == socket.AF_INET:
+			# convert host to network byte order
+			self.addr, = struct.unpack("!L", binary)
+			self.family = family
+			self.plen = 32
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+		elif self.valid and family == socket.AF_INET6:
+			# convert host to network byte order
+			hi, lo = struct.unpack("!QQ", binary)
+			self.addr = (hi << 64) | lo
+			self.family = family
+			self.plen = 128
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+			# if IPv6 address is a IPv4-compatible, make instance a IPv4
+			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
+				self.addr = lo & 0xFFFFFFFFL
+				self.family = socket.AF_INET
+				self.plen = 32
+		else:
+			# string couldn't be converted neither to a IPv4 nor
+			# to a IPv6 address - retain raw input for later use
+			# (e.g. DNS resolution)
+			self.raw = ipstring
+
+	def __repr__(self):
+		return self.ntoa()
+
+	def __str__(self):
+		return self.ntoa()
+
+	def __eq__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not self.valid and not other.valid: return self.raw == other.raw
+		if not self.valid or not other.valid: return False
+		if self.addr != other.addr: return False
+		if self.family != other.family: return False
+		if self.plen != other.plen: return False
+		return True
+
+	def __ne__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not self.valid and not other.valid: return self.raw != other.raw
+		if self.addr != other.addr: return True
+		if self.family != other.family: return True
+		if self.plen != other.plen: return True
+		return False
+
+	def __lt__(self, other):
+		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		return self.family < other.family or self.addr < other.addr
+
+	def __add__(self, other):
+		return "%s%s" % (self, other)
+
+	def __radd__(self, other):
+		return "%s%s" % (other, self)
+
+	def __hash__(self):
+		return hash(self.addr)^hash((self.plen<<16)|self.family)
+
+	def hexdump(self):
+		""" dump the ip address in as a hex sequence in
+			network byte order - for debug purpose
+		"""
+		if self.family == socket.AF_INET:
+			return "%08x" % self.addr
+		elif self.family == socket.AF_INET6:
+			return "%032x" % self.addr
+		else:
+			return ""
+	
+	def ntoa(self):
+		""" represent IP object as text like the depricated 
+			C pendant inet_ntoa() but address family independent
+		"""
+		if self.family == socket.AF_INET:
+			# convert network to host byte order
+			binary = struct.pack("!L", self.addr)
+		elif self.family == socket.AF_INET6:
+			# convert network to host byte order
+			hi = self.addr >> 64
+			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			binary = struct.pack("!QQ", hi, lo)
+		else:
+			return self.getRaw()
+
+		return socket.inet_ntop(self.family, binary)
+
+	def getPTR(self, suffix=""):
+		""" generates the DNS PTR string of the provided IP address object
+			if "suffix" is provided it will be appended as the second and top
+			level reverse domain.
+			if omitted it is implicitely set to the second and top level reverse
+			domain of the according IP address family
+		"""
+		if self.family == socket.AF_INET:
+			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+			if not suffix:
+				suffix = "in-addr.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+
+		elif self.family == socket.AF_INET6:
+			reversed_ip = ".".join(reversed(self.hexdump()))
+			if not suffix:
+				suffix =  "ip6.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+			
+		else:
+			return ""
+
+	def isIPv4(self):
+		""" return true if the IP object is of address family AF_INET
+		"""
+		return True if self.family == socket.AF_INET else False
+
+	def isIPv6(self):
+		""" return true if the IP object is of address family AF_INET6
+		"""
+		return True if self.family == socket.AF_INET6 else False
+
+	def getRaw(self):
+		""" returns the raw attribute - should only be set
+			to a non-empty string if prior address conversion
+			wasn't possible
+		"""
+		return self.raw
+
+	def isValidIP(self):
+		""" returns true if the IP object has been created
+			from a valid IP address or false if not
+		"""
+		return self.valid
+
+	
+	def isInNet(self, net):
+		""" returns true if the IP object is in the provided
+			network (object)
+		"""
+		if self.family != net.family:
+			return False
+
+		if self.family == socket.AF_INET:
+			mask = ~(0xFFFFFFFFL >> net.plen)
+
+		elif self.family == socket.AF_INET6:
+			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
+		else:
+			return False
+		
+		if self.addr & mask == net.addr:
+			return True
+
+		return False
+
+		
+	@staticmethod
+	def masktoplen(maskstr):
+		""" converts mask string to prefix length
+			only used for IPv4 masks
+		"""
+		mask = IPAddr(maskstr)
+		plen = 0
+		while mask.addr:
+			mask.addr = (mask.addr << 1) & 0xFFFFFFFFL
+			plen += 1
+		return plen
+
+
+	@staticmethod
+	def searchIP(text):
+		""" Search if an IP address if directly available and return
+			it.
+		"""
+		match = IPAddr.IP_CRE.match(text)
+		if match:
+			return match
+		else:
+			match = IPAddr.IP6_CRE.match(text)
+			if match:
+				return match
+			else:
+				return None
+

From 130874434879d2d59b4b81cd20c67be036083f77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 12:53:34 +0200
Subject: [PATCH 036/623] meantime commit: code review, simplification,
 pythonization, etc.

---
 fail2ban/server/filter.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index da5fc1ea..1b46aa2a 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -433,7 +433,7 @@ class Filter(JailThread):
 				logSys.debug("Ignore line since time %s < %s - %s", 
 					unixTime, MyTime.time(), self.getFindTime())
 				break
-			if self.inIgnoreIPList(ip.ntoa(), log_ignore=True):
+			if self.inIgnoreIPList(ip, log_ignore=True):
 				continue
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
@@ -530,8 +530,7 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							ipaddr = IPAddr(host)
-							failList.append([failRegexIndex, ipaddr, date,
+							failList.append([failRegexIndex, IPAddr(host), date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
 								break
@@ -539,9 +538,8 @@ class Filter(JailThread):
 							ipMatch = DNSUtils.textToIp(host, self.__useDns)
 							if ipMatch:
 								for ip in ipMatch:
-									ipaddr = IPAddr(ip)
-									failList.append([failRegexIndex, ipaddr, 
-										 date, failRegex.getMatchedLines()])
+									failList.append([failRegexIndex, ip, date,
+										 failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable

From a0938286023ea09f1c668ecb526f3f227e4a3d5e Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 29 Feb 2016 20:38:40 +0100
Subject: [PATCH 037/623] Make ignoreip checking address family idependent

---
 fail2ban/server/filter.py | 44 +++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 1b46aa2a..af23a1d2 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -336,7 +336,22 @@ class Filter(JailThread):
 	# when finding failures. CIDR mask and DNS are also accepted.
 	# @param ip IP address to ignore
 
-	def addIgnoreIP(self, ip):
+	def addIgnoreIP(self, ipstr):
+		# An empty string is always false
+		if ipstr == "":
+			return
+		s = ipstr.split('/', 1)
+		# IP address without CIDR mask
+		if len(s) == 1:
+			s.insert(1, -1) # <0 means no CIDR
+		elif "." in s[1]: # 255.255.255.0 style mask
+			s[1] = IPAddr.masktoplen(s[1])
+		s[1] = long(s[1])
+		
+		# Create IP address object
+		ip = IPAddr(s[0], s[1])
+
+		# log and append to ignore list
 		logSys.debug("Add " + ip + " to ignore list")
 		self.__ignoreIpList.append(ip)
 
@@ -356,34 +371,22 @@ class Filter(JailThread):
 	#
 	# Check if the given IP address matches an IP address/DNS or a CIDR
 	# mask in the ignore list.
-	# @param ip IP address
+	# @param ip IP address object
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
-		for i in self.__ignoreIpList:
-			# An empty string is always false
-			if i == "":
-				continue
-			s = i.split('/', 1)
-			# IP address without CIDR mask
-			if len(s) == 1:
-				s.insert(1, '32')
-			elif "." in s[1]: # 255.255.255.0 style mask
-				s[1] = len(re.search(
-					"(?<=b)1+", bin(DNSUtils.addr2bin(s[1]))).group())
-			s[1] = long(s[1])
-			try:
-				a = DNSUtils.addr2bin(s[0], cidr=s[1])
-				b = DNSUtils.addr2bin(ip, cidr=s[1])
-			except Exception:
+		for net in self.__ignoreIpList:
+			# if it isn't a valid IP address, try DNS resolution
+			if not net.isValidIP() and net.getRaw() != "":
 				# Check if IP in DNS
-				ips = DNSUtils.dnsToIp(i)
+				ips = DNSUtils.dnsToIp(net.getRaw())
 				if ip in ips:
 					self.logIgnoreIp(ip, log_ignore, ignore_source="dns")
 					return True
 				else:
 					continue
-			if a == b:
+			# check if the IP is covered by ignore IP
+			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
 				return True
 
@@ -391,6 +394,7 @@ class Filter(JailThread):
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
 			logSys.debug('ignore command: ' + command)
 			ret_ignore = CommandAction.executeCmd(command)
+
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
 			return ret_ignore
 

From 3893a6b780ad8271f5a10e452ccd6a6d652d51ed Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 13:50:37 +0200
Subject: [PATCH 038/623] meantime commit: code review, simplification,
 pythonization, etc.

---
 fail2ban/server/filter.py        | 3 ++-
 fail2ban/tests/servertestcase.py | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index af23a1d2..e19b322e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -375,6 +375,8 @@ class Filter(JailThread):
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
+		if isinstance(ip, basestring):
+			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
 			# if it isn't a valid IP address, try DNS resolution
 			if not net.isValidIP() and net.getRaw() != "":
@@ -394,7 +396,6 @@ class Filter(JailThread):
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
 			logSys.debug('ignore command: ' + command)
 			ret_ignore = CommandAction.executeCmd(command)
-
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
 			return ret_ignore
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 96734262..d9db6c47 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -125,14 +125,14 @@ class TransmitterBase(unittest.TestCase):
 			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdAdd, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdDel, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 
 	def jailAddDelRegexTest(self, cmd, inValues, outValues, jail):
 		cmdAdd = "add" + cmd

From a7570376717c07e783bb97ee75893f9b0713b0eb Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 29 Feb 2016 20:54:58 +0100
Subject: [PATCH 039/623] Make DNS resolution IP address family idependent

---
 fail2ban/server/filter.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e19b322e..deb47a4b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1012,18 +1012,22 @@ class DNSUtils:
 			Thanks to Kevin Drapel.
 		"""
 		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		v = DNSUtils.CACHE_nameToIp.get(dns)
-		if v is not None: 
-			return v
-		# retrieve ip (todo: use AF_INET6 for IPv6)
+		ips = DNSUtils.CACHE_nameToIp.get(dns)
+		if ips is not None: 
+			return ips
+		# retrieve ips
 		try:
-			v = set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
+			ips = list()
+			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
+				ip = IPAddr(result[4][0])
+				if ip.isValidIP():
+					ips.append(ip)
 		except socket.error, e:
 			# todo: make configurable the expired time of cache entry:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			v = list()
-		DNSUtils.CACHE_nameToIp.set(dns, v)
-		return v
+			ips = list()
+		DNSUtils.CACHE_nameToIp.set(dns, ips)
+		return ips
 
 	@staticmethod
 	def ipToName(ip):
@@ -1033,7 +1037,7 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			v = socket.gethostbyaddr(ip)[0]
+			v = socket.gethostbyaddr(ip.ntoa())[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -1068,11 +1072,11 @@ class DNSUtils:
 		"""
 		ipList = list()
 		# Search for plain IP
-		plainIP = DNSUtils.searchIP(text)
+		plainIP = IPAddr.searchIP(text)
 		if not plainIP is None:
-			plainIPStr = plainIP.group(0)
-			if DNSUtils.isValidIP(plainIPStr):
-				ipList.append(plainIPStr)
+			ip = IPAddr(plainIP.group(0))
+			if ip.isValidIP():
+				ipList.append(ip)
 
 		# If we are allowed to resolve -- give it a try if nothing was found
 		if useDns in ("yes", "warn") and not ipList:

From 07c9f38e4529cc45e3336d30e66d3c4c195fefff Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 14:02:03 +0200
Subject: [PATCH 040/623] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/server/filter.py        | 7 +++++--
 fail2ban/tests/filtertestcase.py | 6 +++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index deb47a4b..3682125c 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -375,7 +375,7 @@ class Filter(JailThread):
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
-		if isinstance(ip, basestring):
+		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
 			# if it isn't a valid IP address, try DNS resolution
@@ -1037,7 +1037,10 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			v = socket.gethostbyaddr(ip.ntoa())[0]
+			if not isinstance(ip, IPAddr):
+				v = socket.gethostbyaddr(ip)[0]
+			else:
+				v = socket.gethostbyaddr(ip.ntoa())[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ae22b88f..b5a772d9 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1314,9 +1314,9 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1328,7 +1328,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		for s in hostnames:
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
-				self.assertEqual(res, ['93.184.216.34'])
+				self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 

From 85b895178b8a6dd9ce7c9086c8c451b72efdb4c1 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Wed, 2 Mar 2016 06:56:57 +0100
Subject: [PATCH 041/623] change IP address string to object handling part 1

# Conflicts:
#	fail2ban/server/filter.py
---
 fail2ban/server/actions.py    |  9 ++++++---
 fail2ban/server/banmanager.py |  6 +++---
 fail2ban/server/database.py   | 16 ++++++++++------
 fail2ban/server/filter.py     | 18 ++++++++++++------
 fail2ban/server/ticket.py     |  3 ---
 5 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 5469722f..428d1ccf 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -42,6 +42,7 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
+from .filter import IPAddr
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -180,7 +181,7 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def removeBannedIP(self, ip):
+	def removeBannedIP(self, ipstr):
 		"""Removes banned IP calling actions' unban method
 
 		Remove a banned IP now, rather than waiting for it to expire,
@@ -188,14 +189,16 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ip : str
-			The IP address to unban
+		ipstr : str
+			The IP address string to unban
 
 		Raises
 		------
 		ValueError
 			If `ip` is not banned
 		"""
+		# Create new IPAddr object from IP string
+		ip = IPAddr(ipstr)
 		# Always delete ip from database (also if currently not banned)
 		if self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 0ee028ef..67ae1b71 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -152,9 +152,9 @@ class BanManager:
 			for banData in self.__banList:
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
-				# TODO: IPv6 compatibility
-				reversed_ip = ".".join(reversed(ip.split(".")))
-				question = "%s.origin.asn.cymru.com" % reversed_ip
+				question = ip.getPTR("origin.asn.cymru.com" if ip.isIPv4()
+						else "origin6.asn.cymru.com"
+				)
 				try:
 					answers = dns.resolver.query(question, "TXT")
 					for rdata in answers:
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index b7fd4d47..e5e51992 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -32,6 +32,7 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
+from .filter import IPAddr
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -422,7 +423,7 @@ class Fail2BanDb(object):
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ticket.getIP(), int(round(ticket.getTime())),
+			(jail.name, ticket.getIP().ntoa(), int(round(ticket.getTime())),
 				ticket.getData()))
 
 	@commitandrollback
@@ -436,7 +437,7 @@ class Fail2BanDb(object):
 		ip : str
 			IP to be removed.
 		"""
-		queryArgs = (jail.name, ip);
+		queryArgs = (jail.name, ip.ntoa());
 		cur.execute(
 			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
 			queryArgs);
@@ -454,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip)
+			queryArgs.append(ip.ntoa())
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)
@@ -470,7 +471,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now.  Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : str
+		ip : IPAddr object
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -479,7 +480,8 @@ class Fail2BanDb(object):
 			List of `Ticket`s for bans stored in database.
 		"""
 		tickets = []
-		for ip, timeofban, data in self._getBans(**kwargs):
+		for ipstr, timeofban, data in self._getBans(**kwargs):
+			ip = IPAddr(ipstr)
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban))
 			tickets[-1].setData(data)
@@ -499,7 +501,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now. Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : str
+		ip : IPAddr object
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -520,6 +522,8 @@ class Fail2BanDb(object):
 			ticket = None
 
 			results = list(self._getBans(ip=ip, jail=jail, bantime=bantime))
+			# Convert IP strings to IPAddr objects
+			results = map(lambda i:(IPAddr(i[0]),)+i[1:], results)
 			if results:
 				prev_banip = results[0][0]
 				matches = []
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 3682125c..371a13d7 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -306,13 +306,19 @@ class Filter(JailThread):
 	def getIgnoreCommand(self):
 		return self.__ignoreCommand
 
+	##
+	# create new IPAddr object from IP address string
+	def newIP(self, ipstr):
+		return IPAddr(ipstr)
+
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
 	#
 	# to enable banip fail2ban-client BAN command
 
-	def addBannedIP(self, ip):
+	def addBannedIP(self, ipstr):
+		ip = IPAddr(ipstr)
 		if self.inIgnoreIPList(ip):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
@@ -540,11 +546,11 @@ class Filter(JailThread):
 							if not checkAllRegex:
 								break
 						else:
-							ipMatch = DNSUtils.textToIp(host, self.__useDns)
-							if ipMatch:
-								for ip in ipMatch:
-									failList.append([failRegexIndex, ip, date,
-										 failRegex.getMatchedLines()])
+							ips = DNSUtils.textToIp(host, self.__useDns)
+							if ips:
+								for ip in ips:
+									failList.append([failRegexIndex, ip, 
+										 date, failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index b3eea052..7b7eb908 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -72,9 +72,6 @@ class Ticket:
 			return False
 
 	def setIP(self, value):
-		if isinstance(value, basestring):
-			# guarantee using regular str instead of unicode for the IP
-			value = str(value)
 		self.__ip = value
 	
 	def getIP(self):

From afe1f73af27503972a4843a770adf3e46f207cde Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 May 2016 19:44:52 +0200
Subject: [PATCH 042/623] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed) unnecessarily code aggravation with
 explicit converting reverted - implicit converting inside internal functions
 if not IPAddr object;

---
 fail2ban/server/actions.py            |   8 +-
 fail2ban/server/database.py           |  21 ++--
 fail2ban/server/failmanager.py        |   9 ++
 fail2ban/server/filter.py             | 100 +++++++++--------
 fail2ban/server/ticket.py             |   4 +
 fail2ban/tests/failmanagertestcase.py |   3 +-
 fail2ban/tests/filtertestcase.py      | 153 ++++++++++++++++++++++----
 7 files changed, 212 insertions(+), 86 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 428d1ccf..1b72af91 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -181,7 +181,7 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def removeBannedIP(self, ipstr):
+	def removeBannedIP(self, ip):
 		"""Removes banned IP calling actions' unban method
 
 		Remove a banned IP now, rather than waiting for it to expire,
@@ -189,16 +189,14 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ipstr : str
-			The IP address string to unban
+		ip : str or IPAddr
+			The IP address  to unban
 
 		Raises
 		------
 		ValueError
 			If `ip` is not banned
 		"""
-		# Create new IPAddr object from IP string
-		ip = IPAddr(ipstr)
 		# Always delete ip from database (also if currently not banned)
 		if self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index e5e51992..6a3d87c3 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -32,7 +32,6 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
-from .filter import IPAddr
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -412,18 +411,19 @@ class Fail2BanDb(object):
 		ticket : BanTicket
 			Ticket of the ban to be added.
 		"""
+		ip = str(ticket.getIP())
 		try:
-			del self._bansMergedCache[(ticket.getIP(), jail)]
+			del self._bansMergedCache[(ip, jail)]
 		except KeyError:
 			pass
 		try:
-			del self._bansMergedCache[(ticket.getIP(), None)]
+			del self._bansMergedCache[(ip, None)]
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ticket.getIP().ntoa(), int(round(ticket.getTime())),
+			(jail.name, ip, int(round(ticket.getTime())),
 				ticket.getData()))
 
 	@commitandrollback
@@ -437,7 +437,7 @@ class Fail2BanDb(object):
 		ip : str
 			IP to be removed.
 		"""
-		queryArgs = (jail.name, ip.ntoa());
+		queryArgs = (jail.name, str(ip));
 		cur.execute(
 			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
 			queryArgs);
@@ -455,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip.ntoa())
+			queryArgs.append(ip)
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)
@@ -471,7 +471,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now.  Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : IPAddr object
+		ip : str
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -480,8 +480,7 @@ class Fail2BanDb(object):
 			List of `Ticket`s for bans stored in database.
 		"""
 		tickets = []
-		for ipstr, timeofban, data in self._getBans(**kwargs):
-			ip = IPAddr(ipstr)
+		for ip, timeofban, data in self._getBans(**kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban))
 			tickets[-1].setData(data)
@@ -501,7 +500,7 @@ class Fail2BanDb(object):
 			Ban time in seconds, such that bans returned would still be
 			valid now. Negative values are equivalent to `None`.
 			Default `None`; no limit.
-		ip : IPAddr object
+		ip : str
 			IP Address to filter bans by. Default `None`; all IPs.
 
 		Returns
@@ -522,8 +521,6 @@ class Fail2BanDb(object):
 			ticket = None
 
 			results = list(self._getBans(ip=ip, jail=jail, bantime=bantime))
-			# Convert IP strings to IPAddr objects
-			results = map(lambda i:(IPAddr(i[0]),)+i[1:], results)
 			if results:
 				prev_banip = results[0][0]
 				matches = []
diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index ae97b36a..b342b280 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -54,6 +54,15 @@ class FailManager:
 		with self.__lock:
 			return self.__failTotal
 	
+	def getFailCount(self):
+		# may be slow on large list of failures, should be used for test purposes only...
+		with self.__lock:
+			return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()])
+
+	def getFailTotal(self):
+		with self.__lock:
+			return self.__failTotal
+
 	def setMaxRetry(self, value):
 		self.__maxRetry = value
 	
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 371a13d7..04f34821 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -306,19 +306,15 @@ class Filter(JailThread):
 	def getIgnoreCommand(self):
 		return self.__ignoreCommand
 
-	##
-	# create new IPAddr object from IP address string
-	def newIP(self, ipstr):
-		return IPAddr(ipstr)
-
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
 	#
 	# to enable banip fail2ban-client BAN command
 
-	def addBannedIP(self, ipstr):
-		ip = IPAddr(ipstr)
+	def addBannedIP(self, ip):
+		if not isinstance(ip, IPAddr):
+			ip = IPAddr(ip)
 		if self.inIgnoreIPList(ip):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
@@ -358,11 +354,11 @@ class Filter(JailThread):
 		ip = IPAddr(s[0], s[1])
 
 		# log and append to ignore list
-		logSys.debug("Add " + ip + " to ignore list")
+		logSys.debug("Add %r to ignore list (%r, %r)", ip, s[0], s[1])
 		self.__ignoreIpList.append(ip)
 
 	def delIgnoreIP(self, ip):
-		logSys.debug("Remove " + ip + " from ignore list")
+		logSys.debug("Remove %r from ignore list", ip)
 		self.__ignoreIpList.remove(ip)
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
@@ -384,18 +380,9 @@ class Filter(JailThread):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
 		for net in self.__ignoreIpList:
-			# if it isn't a valid IP address, try DNS resolution
-			if not net.isValidIP() and net.getRaw() != "":
-				# Check if IP in DNS
-				ips = DNSUtils.dnsToIp(net.getRaw())
-				if ip in ips:
-					self.logIgnoreIp(ip, log_ignore, ignore_source="dns")
-					return True
-				else:
-					continue
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
-				self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
+				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValidIP() else "dns"))
 				return True
 
 		if self.__ignoreCommand:
@@ -1006,8 +993,6 @@ from .utils import Utils
 
 class DNSUtils:
 
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
 	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
@@ -1053,17 +1038,6 @@ class DNSUtils:
 		DNSUtils.CACHE_ipToName.set(ip, v)
 		return v
 
-	@staticmethod
-	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
-		"""
-		match = DNSUtils.IP_CRE.match(text)
-		if match:
-			return match
-		else:
-			return None
-
 	@staticmethod
 	def isValidIP(string):
 		""" Return true if str is a valid IP
@@ -1082,7 +1056,7 @@ class DNSUtils:
 		ipList = list()
 		# Search for plain IP
 		plainIP = IPAddr.searchIP(text)
-		if not plainIP is None:
+		if plainIP is not None:
 			ip = IPAddr(plainIP.group(0))
 			if ip.isValidIP():
 				ipList.append(ip)
@@ -1122,7 +1096,7 @@ class DNSUtils:
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
 
-class IPAddr:
+class IPAddr(object):
 	""" provide functions to handle IPv4 and IPv6 addresses 
 	"""
 
@@ -1136,8 +1110,22 @@ class IPAddr:
 	valid = False
 	raw = ""
 
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
+
 	# object methods
-	def __init__(self, ipstring, cidr=-1):
+	def __init(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -1193,7 +1181,9 @@ class IPAddr:
 		return self.ntoa()
 
 	def __eq__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
 		if not self.valid and not other.valid: return self.raw == other.raw
 		if not self.valid or not other.valid: return False
 		if self.addr != other.addr: return False
@@ -1202,7 +1192,9 @@ class IPAddr:
 		return True
 
 	def __ne__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return True
+			other = IPAddr(other)
 		if not self.valid and not other.valid: return self.raw != other.raw
 		if self.addr != other.addr: return True
 		if self.family != other.family: return True
@@ -1210,7 +1202,9 @@ class IPAddr:
 		return False
 
 	def __lt__(self, other):
-		other = other if isinstance(other, IPAddr) else IPAddr(other)
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
@@ -1220,7 +1214,9 @@ class IPAddr:
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		return hash(self.addr)^hash((self.plen<<16)|self.family)
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa())
+		#return hash(self.addr)^hash((self.plen<<16)|self.family)
 
 	def hexdump(self):
 		""" dump the ip address in as a hex sequence in
@@ -1302,6 +1298,11 @@ class IPAddr:
 		""" returns true if the IP object is in the provided
 			network (object)
 		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValidIP() and net.getRaw() != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.getRaw())
+
 		if self.family != net.family:
 			return False
 
@@ -1318,18 +1319,27 @@ class IPAddr:
 
 		return False
 
+	@property
+	def maskplen(self):
+		plen = 0
+		if (hasattr(self, '_maskplen')):
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+
 		
 	@staticmethod
 	def masktoplen(maskstr):
 		""" converts mask string to prefix length
 			only used for IPv4 masks
 		"""
-		mask = IPAddr(maskstr)
-		plen = 0
-		while mask.addr:
-			mask.addr = (mask.addr << 1) & 0xFFFFFFFFL
-			plen += 1
-		return plen
+		return IPAddr(maskstr).maskplen
 
 
 	@staticmethod
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 7b7eb908..3307a6c9 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -72,6 +72,10 @@ class Ticket:
 			return False
 
 	def setIP(self, value):
+		# guarantee using IPAddr instead of unicode, str for the IP
+		if isinstance(value, basestring):
+			from .filter import IPAddr
+			value = IPAddr(value)
 		self.__ip = value
 	
 	def getIP(self):
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index a8a71723..36bc87a3 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -28,6 +28,7 @@ import unittest
 
 from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
+from ..server.filter import IPAddr
 from ..server.ticket import FailTicket
 
 
@@ -140,7 +141,7 @@ class AddFailure(unittest.TestCase):
 		#ticket = FailTicket('193.168.0.128', None)
 		ticket = self.__failManager.toBan()
 		self.assertEqual(ticket.getIP(), "193.168.0.128")
-		self.assertTrue(isinstance(ticket.getIP(), str))
+		self.assertTrue(isinstance(ticket.getIP(), (str, IPAddr)))
 
 		# finish with rudimentary tests of the ticket
 		# verify consistent str
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index b5a772d9..7198185a 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -38,7 +38,7 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils
+from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils, IPAddr
 from ..server.failmanager import FailManagerEmpty
 from ..server.mytime import MyTime
 from ..server.utils import Utils
@@ -154,19 +154,41 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None):
 
 	Test filter to contain target ticket
 	"""
+	# one or multiple tickets:
+	if not isinstance(output[0], (tuple,list)):
+		tickcount = 1
+		failcount = (count if count else output[1])
+	else:
+		tickcount = len(output)
+		failcount = (count if count else sum((o[1] for o in output)))
+
+	found = []
 	if isinstance(filter_, DummyJail):
 		# get fail ticket from jail
-		found = _ticket_tuple(filter_.getFailTicket())
+		found.append(_ticket_tuple(filter_.getFailTicket()))
 	else:
 		# when we are testing without jails
 		# wait for failures (up to max time)
 		Utils.wait_for(
-			lambda: filter_.failManager.getFailTotal() >= (count if count else output[1]),
+			lambda: filter_.failManager.getFailCount() >= (tickcount, failcount),
 			_maxWaitTime(10))
-		# get fail ticket from filter
-		found = _ticket_tuple(filter_.failManager.toBan())
+		# get fail ticket(s) from filter
+		while tickcount:
+			try:
+				found.append(_ticket_tuple(filter_.failManager.toBan()))
+			except FailManagerEmpty:
+				break
+			tickcount -= 1
 
-	_assert_equal_entries(utest, found, output, count)
+	if not isinstance(output[0], (tuple,list)):
+		utest.assertEqual(len(found), 1)
+		_assert_equal_entries(utest, found[0], output, count)
+	else:
+		# sort by string representation of ip (multiple failures with different ips):
+		found = sorted(found, key=lambda x: str(x))
+		output = sorted(output, key=lambda x: str(x))
+		for f, o in zip(found, output):
+			_assert_equal_entries(utest, f, o)
 
 
 def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""):
@@ -315,6 +337,10 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.1.255'))
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.0.255'))
 
+	def testWrongIPMask(self):
+		self.filter.addIgnoreIP('192.168.1.0/255.255.0.0')
+		self.assertRaises(ValueError, self.filter.addIgnoreIP, '192.168.1.0/255.255.0.128')
+
 	def testIgnoreInProcessLine(self):
 		setUpMyTime()
 		self.filter.addIgnoreIP('192.168.1.0/25')
@@ -345,16 +371,21 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertNotLogged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED"))
 
 
-class IgnoreIPDNS(IgnoreIP):
+class IgnoreIPDNS(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
-		IgnoreIP.setUp(self)
+		LogCaptureTestCase.setUp(self)
+		self.jail = DummyJail()
+		self.filter = FileFilter(self.jail)
 
 	def testIgnoreIPDNSOK(self):
 		self.filter.addIgnoreIP("www.epfl.ch")
 		self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12"))
+		self.filter.addIgnoreIP("example.com")
+		self.assertTrue(self.filter.inIgnoreIPList("93.184.216.34"))
+		self.assertTrue(self.filter.inIgnoreIPList("2606:2800:220:1:248:1893:25c8:1946"))
 
 	def testIgnoreIPDNSNOK(self):
 		# Test DNS
@@ -1109,18 +1140,18 @@ class GetFailures(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures04(self):
-		output = [('212.41.96.186', 4, 1124013600.0),
-				  ('212.41.96.185', 4, 1124017198.0)]
+		# because of not exact time in testcase04.log (no year), we should always use our test time:
+		self.assertEqual(MyTime.time(), 1124013600)
+		# should find exact 4 failures for *.186 and 2 failures for *.185
+		output = (('212.41.96.186', 4, 1124013600.0),
+				  ('212.41.96.185', 2, 1124013598.0))
 
+		self.filter.setMaxRetry(2)
 		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_04)
 
-		try:
-			for i, out in enumerate(output):
-				_assert_correct_last_attempt(self, self.filter, out)
-		except FailManagerEmpty:
-			pass
+		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailuresWrongChar(self):
 		# write wrong utf-8 char:
@@ -1160,20 +1191,31 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresUseDNS(self):
 		unittest.F2B.SkipIfNoNetwork()
 		# We should still catch failures with usedns = no ;-)
-		output_yes = ('93.184.216.34', 2, 1124013539.0,
-					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
-					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_yes = (
+			('93.184.216.34', 2, 1124013539.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
+			   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			),
+			('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2']
+			),
+		)
 
-		output_no = ('93.184.216.34', 1, 1124013539.0,
-					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_no = (
+			('93.184.216.34', 1, 1124013539.0,
+			  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			)
+		)
 
 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,
 		#				  FileFilter, None, useDns='wrong_value_for_useDns')
 
-		for useDns, output in (('yes',  output_yes),
-							   ('no',   output_no),
-							   ('warn', output_yes)):
+		for useDns, output in (
+			('yes',  output_yes),
+			('no',   output_no),
+			('warn', output_yes)
+		):
 			jail = DummyJail()
 			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.active = True
@@ -1356,6 +1398,71 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.bin2addr(167772160L)
 		self.assertEqual(res, '10.0.0.0')
 
+	def testIPAddr_Equal6(self):
+		self.assertEqual(
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:0:0')
+		)
+
+	def testIPAddr_Compare(self):
+		ip4 = [
+			IPAddr('93.184.0.1'),
+			IPAddr('93.184.216.1'),
+			IPAddr('93.184.216.34')
+		]
+		ip6 = [
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:0'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:1946')
+		]
+		# ip4
+		self.assertNotEqual(ip4[0], None)
+		self.assertTrue(ip4[0] is not None)
+		self.assertFalse(ip4[0] is None)
+		self.assertLess(None, ip4[0])
+		self.assertLess(ip4[0], ip4[1])
+		self.assertLess(ip4[1], ip4[2])
+		self.assertEqual(sorted(reversed(ip4)), ip4)
+		# ip6
+		self.assertNotEqual(ip6[0], None)
+		self.assertTrue(ip6[0] is not None)
+		self.assertFalse(ip6[0] is None)
+		self.assertLess(None, ip6[0])
+		self.assertLess(ip6[0], ip6[1])
+		self.assertLess(ip6[1], ip6[2])
+		self.assertEqual(sorted(reversed(ip6)), ip6)
+		# ip4 vs ip6
+		self.assertNotEqual(ip4[0], ip6[0])
+		self.assertLess(ip4[0], ip6[0])
+		self.assertLess(ip4[2], ip6[2])
+		self.assertEqual(sorted(reversed(ip4+ip6)), ip4+ip6)
+		# hashing (with string as key):
+		d={
+			'93.184.216.34': 'ip4-test', 
+			'2606:2800:220:1:248:1893:25c8:1946': 'ip6-test'
+		}
+		d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()])
+		self.assertTrue(isinstance(d.keys()[0], basestring))
+		self.assertTrue(isinstance(d2.keys()[0], IPAddr))
+		self.assertEqual(d.get(ip4[2], ''), 'ip4-test')
+		self.assertEqual(d.get(ip6[2], ''), 'ip6-test')
+		self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test')
+		self.assertEqual(d2.get(str(ip6[2]), ''), 'ip6-test')
+		# compare with string direct:
+		self.assertEqual(d, d2)
+
+	def testIPAddr_CompareDNS(self):
+		ips = IPAddr('example.com')
+		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))
+		self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips))
+
+	def testIPAddr_Cached(self):
+		ips = [DNSUtils.dnsToIp('example.com'), DNSUtils.dnsToIp('example.com')]
+		for ip1, ip2 in zip(ips, ips):
+			self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('93.184.216.34'); ip2 = IPAddr('93.184.216.34'); self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2))
+
 
 class JailTests(unittest.TestCase):
 

From d125f882d4fd1d1d9e096aa328cdb3eec6abac5e Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 5 Mar 2016 12:02:36 +0100
Subject: [PATCH 043/623] explicitely treat join arguments as string in
 beautifier

---
 fail2ban/client/beautifier.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 812fbe65..308cb035 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -78,13 +78,13 @@ class Beautifier:
 						prefix1 = "   " if n == len(response) - 1 else "|  "
 						for m, res2 in enumerate(res1[1]):
 							prefix2 = prefix1 + ("`-" if m == len(res1[1]) - 1 else "|-")
-							val = " ".join(res2[1]) if isinstance(res2[1], list) else res2[1]
+							val = " ".join(map(str, res2[1])) if isinstance(res2[1], list) else res2[1]
 							msg.append("%s %s:\t%s" % (prefix2, res2[0], val))
 				else:
 					msg = ["Status"]
 					for n, res1 in enumerate(response):
 						prefix1 = "`-" if n == len(response) - 1 else "|-"
-						val = " ".join(res1[1]) if isinstance(res1[1], list) else res1[1]
+						val = " ".join(map(str, res1[1])) if isinstance(res1[1], list) else res1[1]
 						msg.append("%s %s:\t%s" % (prefix1, res1[0], val))
 				msg = "\n".join(msg)
 			elif inC[1] == "syslogsocket":

From db9f3f738f88ef0f7ae535a578055666f1843292 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 23:42:06 +0100
Subject: [PATCH 044/623] add ip6-loopback to default ignoreip statement

---
 config/jail.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index b41ce24c..36c4eecb 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -47,7 +47,7 @@ before = paths-debian.conf
 # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
 # ban a host which matches an address in this list. Several addresses can be
 # defined using space (and/or comma) separator.
-ignoreip = 127.0.0.1/8
+ignoreip = 127.0.0.1/8 ::1
 
 # External command that will take an tagged arguments to ignore, e.g. <ip>,
 # and return true if the IP is to be ignored. False otherwise.

From ce196744d1d509c895995c759ee2f4d014195ac1 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 23:29:12 +0100
Subject: [PATCH 045/623] Update ChangeLog / THANKS entries

---
 ChangeLog | 8 ++++++++
 THANKS    | 1 +
 2 files changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 033cd9ec..0bebb471 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -78,6 +78,14 @@ ver. 0.9.4 (2016/03/08) - for-you-ladies
    * sshd filter got new failregex to match "maximum authentication
      attempts exceeded" (introduced in openssh 6.8)
    * Added filter for Mac OS screen sharing (VNC) daemon
+   * IPv6 support:
+     - IP addresses are now handled as objects rather than strings capable for 
+       handling both address types IPv4 and IPv6
+     - iptables related actions have been amended to support IPv6 specific actions
+       additionally
+     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
+     - pf action for *BSD systems has been improved and supports now also v4 and v6
+     - Name resolution is now working for either address type
 
 - Enhancements:
    * Do not rotate empty log files
diff --git a/THANKS b/THANKS
index 64de43a4..c8a019a6 100644
--- a/THANKS
+++ b/THANKS
@@ -12,6 +12,7 @@ Adrien Clerc
 ache
 ag4ve (Shawn)
 Alasdair D. Campbell
+Alexander Koeppe (IPv6 support)
 Alexandre Perrin (kAworu)
 Amir Caspi
 Amy

From 9ede535a616802434ecb9826ed5c40a6ac81317f Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 15 Mar 2016 00:36:08 +0100
Subject: [PATCH 046/623] remove obsolete IP related code from DNSUtils class

# Conflicts:
#	fail2ban/server/filter.py
---
 fail2ban/server/filter.py | 36 +++---------------------------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 04f34821..baf4bc9b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -981,10 +981,10 @@ class JournalFilter(Filter): # pragma: systemd no cover
 		return []
 
 ##
-# Utils class for DNS and IP handling.
+# Utils class for DNS handling.
+#
+# This class contains only static methods used to handle DNS 
 #
-# This class contains only static methods used to handle DNS and IP
-# addresses.
 
 import socket
 import struct
@@ -1038,17 +1038,6 @@ class DNSUtils:
 		DNSUtils.CACHE_ipToName.set(ip, v)
 		return v
 
-	@staticmethod
-	def isValidIP(string):
-		""" Return true if str is a valid IP
-		"""
-		s = string.split('/', 1)
-		try:
-			socket.inet_aton(s[0])
-			return True
-		except socket.error: # pragma: no cover
-			return False
-
 	@staticmethod
 	def textToIp(text, useDns):
 		""" Return the IP of DNS found in a given text.
@@ -1072,25 +1061,6 @@ class DNSUtils:
 
 		return ipList
 
-	@staticmethod
-	def addr2bin(ipstring, cidr=None):
-		""" Convert a string IPv4 address into binary form.
-		If cidr is supplied, return the network address for the given block
-		"""
-		if cidr is None:
-			return struct.unpack("!L", socket.inet_aton(ipstring))[0]
-		else:
-			MASK = 0xFFFFFFFFL
-			return ~(MASK >> cidr) & MASK & DNSUtils.addr2bin(ipstring)
-
-	@staticmethod
-	def bin2addr(ipbin):
-		""" Convert a binary IPv4 address into string n.n.n.n form.
-		"""
-		return socket.inet_ntoa(struct.pack("!L", ipbin))
-
-
-
 ##
 # Class for IP address handling.
 #

From 6985531e91085c0daa7aa746c909a1293bc91bc2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 9 May 2016 16:23:59 +0200
Subject: [PATCH 047/623] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/server/filter.py        |  9 ++++++---
 fail2ban/tests/filtertestcase.py | 24 ++++++++++--------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index baf4bc9b..b808a6e3 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1102,11 +1102,10 @@ class IPAddr(object):
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
-			except socket.error:
-				continue
-			else: 
 				self.valid = True
 				break
+			except socket.error:
+				continue
 
 		if self.valid and family == socket.AF_INET:
 			# convert host to network byte order
@@ -1178,9 +1177,13 @@ class IPAddr(object):
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 7198185a..371db3bc 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1383,20 +1383,16 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(res, None)
 
 	def testAddr2bin(self):
-		res = DNSUtils.addr2bin('10.0.0.0')
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=None)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=32L)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=32L)
-		self.assertEqual(res, 167772161L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=31L)
-		self.assertEqual(res, 167772160L)
-
-	def testBin2addr(self):
-		res = DNSUtils.bin2addr(167772160L)
-		self.assertEqual(res, '10.0.0.0')
+		res = IPAddr('10.0.0.0')
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=None)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=32L)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.1', cidr=32L)
+		self.assertEqual(res.addr, 167772161L)
+		res = IPAddr('10.0.0.1', cidr=31L)
+		self.assertEqual(res.addr, 167772160L)
 
 	def testIPAddr_Equal6(self):
 		self.assertEqual(

From 8cb4a3f59ef1a8078ef6919ce587bb6ebed66711 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 9 May 2016 17:00:35 +0200
Subject: [PATCH 048/623] move DNTUtils, IPAddr related code to dedicated
 source file ipdns.py (also resolves some cyclic import references)

---
 MANIFEST                                      |   1 +
 .../ignorecommands/apache-fakegooglebot       |   4 +-
 fail2ban/server/filter.py                     | 351 +---------------
 fail2ban/server/ipdns.py                      | 389 ++++++++++++++++++
 fail2ban/server/ticket.py                     |   2 +-
 fail2ban/tests/failmanagertestcase.py         |   2 +-
 fail2ban/tests/filtertestcase.py              |   3 +-
 fail2ban/tests/utils.py                       |   2 +-
 8 files changed, 398 insertions(+), 356 deletions(-)
 create mode 100644 fail2ban/server/ipdns.py

diff --git a/MANIFEST b/MANIFEST
index 4ebf3fad..3a5b4477 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -164,6 +164,7 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
+fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index 19fb5107..86a28eaa 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -14,7 +14,7 @@ def process_args(argv):
 
     ip = argv[1]
 
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
     if not DNSUtils.isValidIP(ip):
        sys.stderr.write("Argument must be a single valid IP. Got: %s\n"
                         % ip)
@@ -23,7 +23,7 @@ def process_args(argv):
 
 def is_googlebot(ip):
     import re
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
 
     host = DNSUtils.ipToName(ip)
     if not host or not re.match('.*\.google(bot)?\.com$', host):
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b808a6e3..49aba907 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -31,6 +31,7 @@ import re
 import sys
 
 from .failmanager import FailManagerEmpty, FailManager
+from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
@@ -980,353 +981,3 @@ class JournalFilter(Filter): # pragma: systemd no cover
 	def getJournalMatch(self, match): # pragma: no cover - Base class, not used
 		return []
 
-##
-# Utils class for DNS handling.
-#
-# This class contains only static methods used to handle DNS 
-#
-
-import socket
-import struct
-from .utils import Utils
-
-
-class DNSUtils:
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
-	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	@staticmethod
-	def dnsToIp(dns):
-		""" Convert a DNS into an IP address using the Python socket module.
-			Thanks to Kevin Drapel.
-		"""
-		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		ips = DNSUtils.CACHE_nameToIp.get(dns)
-		if ips is not None: 
-			return ips
-		# retrieve ips
-		try:
-			ips = list()
-			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
-				ip = IPAddr(result[4][0])
-				if ip.isValidIP():
-					ips.append(ip)
-		except socket.error, e:
-			# todo: make configurable the expired time of cache entry:
-			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			ips = list()
-		DNSUtils.CACHE_nameToIp.set(dns, ips)
-		return ips
-
-	@staticmethod
-	def ipToName(ip):
-		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
-		v = DNSUtils.CACHE_ipToName.get(ip, ())
-		if v != ():
-			return v
-		# retrieve name
-		try:
-			if not isinstance(ip, IPAddr):
-				v = socket.gethostbyaddr(ip)[0]
-			else:
-				v = socket.gethostbyaddr(ip.ntoa())[0]
-		except socket.error, e:
-			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
-			v = None
-		DNSUtils.CACHE_ipToName.set(ip, v)
-		return v
-
-	@staticmethod
-	def textToIp(text, useDns):
-		""" Return the IP of DNS found in a given text.
-		"""
-		ipList = list()
-		# Search for plain IP
-		plainIP = IPAddr.searchIP(text)
-		if plainIP is not None:
-			ip = IPAddr(plainIP.group(0))
-			if ip.isValidIP():
-				ipList.append(ip)
-
-		# If we are allowed to resolve -- give it a try if nothing was found
-		if useDns in ("yes", "warn") and not ipList:
-			# Try to get IP from possible DNS
-			ip = DNSUtils.dnsToIp(text)
-			ipList.extend(ip)
-			if ip and useDns == "warn":
-				logSys.warning("Determined IP using DNS Lookup: %s = %s",
-					text, ipList)
-
-		return ipList
-
-##
-# Class for IP address handling.
-#
-# This class contains methods for handling IPv4 and IPv6 addresses.
-
-class IPAddr(object):
-	""" provide functions to handle IPv4 and IPv6 addresses 
-	"""
-
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
-
-	# object attributes
-	addr = 0
-	family = socket.AF_UNSPEC
-	plen = 0
-	valid = False
-	raw = ""
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
-		ip = IPAddr.CACHE_OBJ.get(args)
-		if ip is not None:
-			return ip
-		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
-		return ip
-
-	# object methods
-	def __init(self, ipstring, cidr=-1):
-		""" initialize IP object by converting IP address string
-			to binary to integer
-		"""
-		for family in [socket.AF_INET, socket.AF_INET6]:
-			try:
-				binary = socket.inet_pton(family, ipstring)
-				self.valid = True
-				break
-			except socket.error:
-				continue
-
-		if self.valid and family == socket.AF_INET:
-			# convert host to network byte order
-			self.addr, = struct.unpack("!L", binary)
-			self.family = family
-			self.plen = 32
-
-			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
-				mask = ~(0xFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
-
-		elif self.valid and family == socket.AF_INET6:
-			# convert host to network byte order
-			hi, lo = struct.unpack("!QQ", binary)
-			self.addr = (hi << 64) | lo
-			self.family = family
-			self.plen = 128
-
-			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
-				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
-
-			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
-				self.addr = lo & 0xFFFFFFFFL
-				self.family = socket.AF_INET
-				self.plen = 32
-		else:
-			# string couldn't be converted neither to a IPv4 nor
-			# to a IPv6 address - retain raw input for later use
-			# (e.g. DNS resolution)
-			self.raw = ipstring
-
-	def __repr__(self):
-		return self.ntoa()
-
-	def __str__(self):
-		return self.ntoa()
-
-	def __eq__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw == other.raw
-		if not self.valid or not other.valid: return False
-		if self.addr != other.addr: return False
-		if self.family != other.family: return False
-		if self.plen != other.plen: return False
-		return True
-
-	def __ne__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return True
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw != other.raw
-		if self.addr != other.addr: return True
-		if self.family != other.family: return True
-		if self.plen != other.plen: return True
-		return False
-
-	def __lt__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		return self.family < other.family or self.addr < other.addr
-
-	def __add__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
-		return "%s%s" % (self, other)
-
-	def __radd__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
-		return "%s%s" % (other, self)
-
-	def __hash__(self):
-		# should be the same as by string (because of possible compare with string):
-		return hash(self.ntoa())
-		#return hash(self.addr)^hash((self.plen<<16)|self.family)
-
-	def hexdump(self):
-		""" dump the ip address in as a hex sequence in
-			network byte order - for debug purpose
-		"""
-		if self.family == socket.AF_INET:
-			return "%08x" % self.addr
-		elif self.family == socket.AF_INET6:
-			return "%032x" % self.addr
-		else:
-			return ""
-	
-	def ntoa(self):
-		""" represent IP object as text like the depricated 
-			C pendant inet_ntoa() but address family independent
-		"""
-		if self.family == socket.AF_INET:
-			# convert network to host byte order
-			binary = struct.pack("!L", self.addr)
-		elif self.family == socket.AF_INET6:
-			# convert network to host byte order
-			hi = self.addr >> 64
-			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
-			binary = struct.pack("!QQ", hi, lo)
-		else:
-			return self.getRaw()
-
-		return socket.inet_ntop(self.family, binary)
-
-	def getPTR(self, suffix=""):
-		""" generates the DNS PTR string of the provided IP address object
-			if "suffix" is provided it will be appended as the second and top
-			level reverse domain.
-			if omitted it is implicitely set to the second and top level reverse
-			domain of the according IP address family
-		"""
-		if self.family == socket.AF_INET:
-			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
-			if not suffix:
-				suffix = "in-addr.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-
-		elif self.family == socket.AF_INET6:
-			reversed_ip = ".".join(reversed(self.hexdump()))
-			if not suffix:
-				suffix =  "ip6.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-			
-		else:
-			return ""
-
-	def isIPv4(self):
-		""" return true if the IP object is of address family AF_INET
-		"""
-		return True if self.family == socket.AF_INET else False
-
-	def isIPv6(self):
-		""" return true if the IP object is of address family AF_INET6
-		"""
-		return True if self.family == socket.AF_INET6 else False
-
-	def getRaw(self):
-		""" returns the raw attribute - should only be set
-			to a non-empty string if prior address conversion
-			wasn't possible
-		"""
-		return self.raw
-
-	def isValidIP(self):
-		""" returns true if the IP object has been created
-			from a valid IP address or false if not
-		"""
-		return self.valid
-
-	
-	def isInNet(self, net):
-		""" returns true if the IP object is in the provided
-			network (object)
-		"""
-		# if it isn't a valid IP address, try DNS resolution
-		if not net.isValidIP() and net.getRaw() != "":
-			# Check if IP in DNS
-			return self in DNSUtils.dnsToIp(net.getRaw())
-
-		if self.family != net.family:
-			return False
-
-		if self.family == socket.AF_INET:
-			mask = ~(0xFFFFFFFFL >> net.plen)
-
-		elif self.family == socket.AF_INET6:
-			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
-		else:
-			return False
-		
-		if self.addr & mask == net.addr:
-			return True
-
-		return False
-
-	@property
-	def maskplen(self):
-		plen = 0
-		if (hasattr(self, '_maskplen')):
-			return self._plen
-		maddr = self.addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
-
-		
-	@staticmethod
-	def masktoplen(maskstr):
-		""" converts mask string to prefix length
-			only used for IPv4 masks
-		"""
-		return IPAddr(maskstr).maskplen
-
-
-	@staticmethod
-	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
-		"""
-		match = IPAddr.IP_CRE.match(text)
-		if match:
-			return match
-		else:
-			match = IPAddr.IP6_CRE.match(text)
-			if match:
-				return match
-			else:
-				return None
-
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
new file mode 100644
index 00000000..e75cd9ed
--- /dev/null
+++ b/fail2ban/server/ipdns.py
@@ -0,0 +1,389 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester"
+__copyright__ = "Copyright (c) 2004-2016 Fail2ban Developers"
+__license__ = "GPL"
+
+import re
+import socket
+import struct
+
+from .utils import Utils
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger(__name__)
+
+
+##
+# Helper functions
+#
+#
+def asip(ip):
+	"""A little helper to guarantee ip being an IPAddr instance"""
+	if isinstance(ip, IPAddr):
+		return ip
+	return IPAddr(ip)
+
+
+##
+# Utils class for DNS handling.
+#
+# This class contains only static methods used to handle DNS 
+#
+class DNSUtils:
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	@staticmethod
+	def dnsToIp(dns):
+		""" Convert a DNS into an IP address using the Python socket module.
+			Thanks to Kevin Drapel.
+		"""
+		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
+		ips = DNSUtils.CACHE_nameToIp.get(dns)
+		if ips is not None: 
+			return ips
+		# retrieve ips
+		try:
+			ips = list()
+			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
+				ip = IPAddr(result[4][0])
+				if ip.isValidIP():
+					ips.append(ip)
+		except socket.error, e:
+			# todo: make configurable the expired time of cache entry:
+			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
+			ips = list()
+		DNSUtils.CACHE_nameToIp.set(dns, ips)
+		return ips
+
+	@staticmethod
+	def ipToName(ip):
+		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
+		v = DNSUtils.CACHE_ipToName.get(ip, ())
+		if v != ():
+			return v
+		# retrieve name
+		try:
+			if not isinstance(ip, IPAddr):
+				v = socket.gethostbyaddr(ip)[0]
+			else:
+				v = socket.gethostbyaddr(ip.ntoa())[0]
+		except socket.error, e:
+			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
+			v = None
+		DNSUtils.CACHE_ipToName.set(ip, v)
+		return v
+
+	@staticmethod
+	def textToIp(text, useDns):
+		""" Return the IP of DNS found in a given text.
+		"""
+		ipList = list()
+		# Search for plain IP
+		plainIP = IPAddr.searchIP(text)
+		if plainIP is not None:
+			ip = IPAddr(plainIP.group(0))
+			if ip.isValidIP():
+				ipList.append(ip)
+
+		# If we are allowed to resolve -- give it a try if nothing was found
+		if useDns in ("yes", "warn") and not ipList:
+			# Try to get IP from possible DNS
+			ip = DNSUtils.dnsToIp(text)
+			ipList.extend(ip)
+			if ip and useDns == "warn":
+				logSys.warning("Determined IP using DNS Lookup: %s = %s",
+					text, ipList)
+
+		return ipList
+
+
+##
+# Class for IP address handling.
+#
+# This class contains methods for handling IPv4 and IPv6 addresses.
+class IPAddr(object):
+	""" provide functions to handle IPv4 and IPv6 addresses 
+	"""
+
+	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
+	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+
+	# object attributes
+	addr = 0
+	family = socket.AF_UNSPEC
+	plen = 0
+	valid = False
+	raw = ""
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
+
+	# object methods
+	def __init(self, ipstring, cidr=-1):
+		""" initialize IP object by converting IP address string
+			to binary to integer
+		"""
+		for family in [socket.AF_INET, socket.AF_INET6]:
+			try:
+				binary = socket.inet_pton(family, ipstring)
+				self.valid = True
+				break
+			except socket.error:
+				continue
+
+		if self.valid and family == socket.AF_INET:
+			# convert host to network byte order
+			self.addr, = struct.unpack("!L", binary)
+			self.family = family
+			self.plen = 32
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+		elif self.valid and family == socket.AF_INET6:
+			# convert host to network byte order
+			hi, lo = struct.unpack("!QQ", binary)
+			self.addr = (hi << 64) | lo
+			self.family = family
+			self.plen = 128
+
+			# mask out host portion if prefix length is supplied
+			if cidr != None and cidr >= 0:
+				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+				self.addr = self.addr & mask
+				self.plen = cidr
+
+			# if IPv6 address is a IPv4-compatible, make instance a IPv4
+			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
+				self.addr = lo & 0xFFFFFFFFL
+				self.family = socket.AF_INET
+				self.plen = 32
+		else:
+			# string couldn't be converted neither to a IPv4 nor
+			# to a IPv6 address - retain raw input for later use
+			# (e.g. DNS resolution)
+			self.raw = ipstring
+
+	def __repr__(self):
+		return self.ntoa()
+
+	def __str__(self):
+		return self.ntoa()
+
+	def __eq__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		if not self.valid and not other.valid: return self.raw == other.raw
+		if not self.valid or not other.valid: return False
+		if self.addr != other.addr: return False
+		if self.family != other.family: return False
+		if self.plen != other.plen: return False
+		return True
+
+	def __ne__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return True
+			other = IPAddr(other)
+		if not self.valid and not other.valid: return self.raw != other.raw
+		if self.addr != other.addr: return True
+		if self.family != other.family: return True
+		if self.plen != other.plen: return True
+		return False
+
+	def __lt__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		return self.family < other.family or self.addr < other.addr
+
+	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (self, other)
+
+	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (other, self)
+
+	def __hash__(self):
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa())
+		#return hash(self.addr)^hash((self.plen<<16)|self.family)
+
+	def hexdump(self):
+		""" dump the ip address in as a hex sequence in
+			network byte order - for debug purpose
+		"""
+		if self.family == socket.AF_INET:
+			return "%08x" % self.addr
+		elif self.family == socket.AF_INET6:
+			return "%032x" % self.addr
+		else:
+			return ""
+	
+	def ntoa(self):
+		""" represent IP object as text like the depricated 
+			C pendant inet_ntoa() but address family independent
+		"""
+		if self.family == socket.AF_INET:
+			# convert network to host byte order
+			binary = struct.pack("!L", self.addr)
+		elif self.family == socket.AF_INET6:
+			# convert network to host byte order
+			hi = self.addr >> 64
+			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			binary = struct.pack("!QQ", hi, lo)
+		else:
+			return self.getRaw()
+
+		return socket.inet_ntop(self.family, binary)
+
+	def getPTR(self, suffix=""):
+		""" generates the DNS PTR string of the provided IP address object
+			if "suffix" is provided it will be appended as the second and top
+			level reverse domain.
+			if omitted it is implicitely set to the second and top level reverse
+			domain of the according IP address family
+		"""
+		if self.family == socket.AF_INET:
+			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+			if not suffix:
+				suffix = "in-addr.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+
+		elif self.family == socket.AF_INET6:
+			reversed_ip = ".".join(reversed(self.hexdump()))
+			if not suffix:
+				suffix =  "ip6.arpa."
+
+			return "%s.%s" % (reversed_ip, suffix)
+			
+		else:
+			return ""
+
+	def isIPv4(self):
+		""" return true if the IP object is of address family AF_INET
+		"""
+		return True if self.family == socket.AF_INET else False
+
+	def isIPv6(self):
+		""" return true if the IP object is of address family AF_INET6
+		"""
+		return True if self.family == socket.AF_INET6 else False
+
+	def getRaw(self):
+		""" returns the raw attribute - should only be set
+			to a non-empty string if prior address conversion
+			wasn't possible
+		"""
+		return self.raw
+
+	def isValidIP(self):
+		""" returns true if the IP object has been created
+			from a valid IP address or false if not
+		"""
+		return self.valid
+
+	
+	def isInNet(self, net):
+		""" returns true if the IP object is in the provided
+			network (object)
+		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValidIP() and net.getRaw() != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.getRaw())
+
+		if self.family != net.family:
+			return False
+
+		if self.family == socket.AF_INET:
+			mask = ~(0xFFFFFFFFL >> net.plen)
+
+		elif self.family == socket.AF_INET6:
+			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
+		else:
+			return False
+		
+		if self.addr & mask == net.addr:
+			return True
+
+		return False
+
+	@property
+	def maskplen(self):
+		plen = 0
+		if (hasattr(self, '_maskplen')):
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+
+		
+	@staticmethod
+	def masktoplen(maskstr):
+		""" converts mask string to prefix length
+			only used for IPv4 masks
+		"""
+		return IPAddr(maskstr).maskplen
+
+
+	@staticmethod
+	def searchIP(text):
+		""" Search if an IP address if directly available and return
+			it.
+		"""
+		match = IPAddr.IP_CRE.match(text)
+		if match:
+			return match
+		else:
+			match = IPAddr.IP6_CRE.match(text)
+			if match:
+				return match
+			else:
+				return None
+
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 3307a6c9..130de4f2 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -27,6 +27,7 @@ __license__ = "GPL"
 import sys
 
 from ..helpers import getLogger
+from .ipdns import IPAddr
 from .mytime import MyTime
 
 # Gets the instance of the logger.
@@ -74,7 +75,6 @@ class Ticket:
 	def setIP(self, value):
 		# guarantee using IPAddr instead of unicode, str for the IP
 		if isinstance(value, basestring):
-			from .filter import IPAddr
 			value = IPAddr(value)
 		self.__ip = value
 	
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 36bc87a3..6e7bf367 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -28,7 +28,7 @@ import unittest
 
 from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
-from ..server.filter import IPAddr
+from ..server.ipdns import IPAddr
 from ..server.ticket import FailTicket
 
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 371db3bc..b5a9b8d5 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -38,8 +38,9 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils, IPAddr
+from ..server.filter import Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
+from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 0f328564..756dcc4f 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -32,7 +32,7 @@ import unittest
 from StringIO import StringIO
 
 from ..helpers import getLogger
-from ..server.filter import DNSUtils
+from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 # for action_d.test_smtp :

From 4274ae89c630e478dbf5e9c6dd55ed8dc9fe5d68 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 9 Apr 2016 00:46:17 +0200
Subject: [PATCH 049/623] Quite little adjustments on tests and return value

# Conflicts:
#	fail2ban/server/ipdns.py
---
 fail2ban/server/ipdns.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index e75cd9ed..b36b5cb1 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -303,12 +303,12 @@ class IPAddr(object):
 	def isIPv4(self):
 		""" return true if the IP object is of address family AF_INET
 		"""
-		return True if self.family == socket.AF_INET else False
+		return self.family == socket.AF_INET
 
 	def isIPv6(self):
 		""" return true if the IP object is of address family AF_INET6
 		"""
-		return True if self.family == socket.AF_INET6 else False
+		return self.family == socket.AF_INET6
 
 	def getRaw(self):
 		""" returns the raw attribute - should only be set

From dbd7e347b19d5ca0506ddc8ae2f6c25d4734e711 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sat, 23 Apr 2016 10:08:44 +0200
Subject: [PATCH 050/623] new testcase to test beautifier code

---
 fail2ban/tests/clientbeautifiertestcase.py | 111 +++++++++++++++++++++
 fail2ban/tests/utils.py                    |   5 +
 2 files changed, 116 insertions(+)
 create mode 100644 fail2ban/tests/clientbeautifiertestcase.py

diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
new file mode 100644
index 00000000..d14030ea
--- /dev/null
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -0,0 +1,111 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Alexander Koeppe"
+__copyright__ = "Copyright (c) 2016 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
+__license__ = "GPL"
+
+import unittest
+
+from ..client.beautifier import Beautifier
+from ..version import version
+from ..ipaddr import IPAddr
+
+class BeautifierTest(unittest.TestCase):
+
+	def setUp(self):
+		""" Call before every test case """
+		self.b = Beautifier()
+
+	def tearDown(self):
+		""" Call after every test case """
+
+	def testGetInputCmd(self):
+		cmd = ["test"]
+		self.b.setInputCmd(cmd)
+		self.assertEqual(self.b.getInputCmd(), cmd)
+
+	def testPing(self):
+		self.b.setInputCmd(["ping"])
+		self.assertEqual(self.b.beautify("pong"), "Server replied: pong")
+	
+	def testVersion(self):
+		self.b.setInputCmd(["version"])
+		self.assertEqual(self.b.beautify(version), version)
+
+	def testAddJail(self):
+		self.b.setInputCmd(["add"])
+		self.assertEqual(self.b.beautify("ssh"), "Added jail ssh")
+
+	def testStartJail(self):
+		self.b.setInputCmd(["start"])
+		self.assertEqual(self.b.beautify(None), "Jail started")
+
+	def testFlushLogs(self):
+		self.b.setInputCmd(["flushlogs"])
+		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
+
+	def testStopJail(self):
+		self.b.setInputCmd(["stop", "ssh"])
+		self.assertEqual(self.b.beautify(None), "Jail stopped")
+
+	def testShutdown(self):
+		self.b.setInputCmd(["stop"])
+		self.assertEqual(self.b.beautify(None), "Shutdown successful")
+
+	def testStatus(self):
+		self.b.setInputCmd(["status"])
+		response = (("Number of jails", 0), ("Jail list", ["ssh", "exim4"]))
+		output = "Status\n|- Number of jails:\t0\n`- Jail list:\tssh exim4"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["status", "ssh"])
+		response = (
+					("Filter", [
+							("Currently failed", 0),
+							("Total failed", 0),
+							("File list", "/var/log/auth.log")
+						]
+					),
+					("Actions", [
+							("Currently banned", 3),
+							("Total banned", 3),
+							("Banned IP list", [
+									IPAddr("192.168.0.1"),
+									IPAddr("::ffff:10.2.2.1"),
+									IPAddr("2001:db8::1")
+								]
+							)
+						]
+					)
+				)
+		output = """Status for the jail: ssh
+|- Filter
+|  |- Currently failed:	0
+|  |- Total failed:	0
+|  `- File list:	/var/log/auth.log
+`- Actions
+   |- Currently banned:	3
+   |- Total banned:	3
+   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"""
+
+		self.assertEqual(self.b.beautify(response), output)
+
+
+	
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 756dcc4f..4c429105 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -133,6 +133,7 @@ def gatherTests(regexps=None, opts=None):
 	# Import all the test cases here instead of a module level to
 	# avoid circular imports
 	from . import banmanagertestcase
+	from . import clientbeautifiertestcase
 	from . import clientreadertestcase
 	from . import tickettestcase
 	from . import failmanagertestcase
@@ -187,6 +188,10 @@ def gatherTests(regexps=None, opts=None):
 		tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo))
 	except ImportError: # pragma: no cover
 		pass
+	
+	# ClientBeautifier
+	tests.addTest(unittest.makeSuite(clientbeautifiertestcase.BeautifierTest))
+
 	# ClientReaders
 	tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))

From 5040c95bdb51fb230cf133611fca1a651d4b89a0 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 10 May 2016 21:17:09 -0400
Subject: [PATCH 051/623] DOC: stub for 0.10.0 release changelog

---
 ChangeLog | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 033cd9ec..bf9df1b5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,20 @@
 Fail2Ban: Changelog
 ===================
 
+ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
+-----------
+
+TODO: list all the fixes/enhancements/etc which came with the PR #1346
+
+- Fixes:
+
+
+- New Features:
+
+
+- Enhancements:
+
+
 ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
 -----------
 

From c1a54974e921206708ce033f684188c2d9b5d12e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Sat, 23 Apr 2016 20:05:27 -0400
Subject: [PATCH 052/623] RF/ENH: 1st wave of IPAddr pythonization -
 properties, logical statements, etc

# Conflicts:
#	fail2ban/server/ipdns.py
---
 MANIFEST                                   |   1 +
 fail2ban/server/banmanager.py              |   5 +-
 fail2ban/server/filter.py                  |   2 +-
 fail2ban/server/ipdns.py                   | 284 +++++++++------------
 fail2ban/tests/clientbeautifiertestcase.py |   2 -
 5 files changed, 126 insertions(+), 168 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index 3a5b4477..11f635c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -201,6 +201,7 @@ fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
 fail2ban/tests/banmanagertestcase.py
 fail2ban/tests/clientreadertestcase.py
+fail2ban/tests/clientbeautifiertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
 fail2ban/tests/config/filter.d/simple.conf
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 67ae1b71..afa70685 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -152,8 +152,9 @@ class BanManager:
 			for banData in self.__banList:
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
-				question = ip.getPTR("origin.asn.cymru.com" if ip.isIPv4()
-						else "origin6.asn.cymru.com"
+				question = ip.getPTR(
+					"origin.asn.cymru.com" if ip.isIPv4
+					else "origin6.asn.cymru.com"
 				)
 				try:
 					answers = dns.resolver.query(question, "TXT")
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 49aba907..4bbf7996 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -383,7 +383,7 @@ class Filter(JailThread):
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
-				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValidIP() else "dns"))
+				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
 				return True
 
 		if self.__ignoreCommand:
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index b36b5cb1..744a8a73 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -17,13 +17,13 @@
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester"
+__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004-2016 Fail2ban Developers"
 __license__ = "GPL"
 
-import re
 import socket
 import struct
+import re
 
 from .utils import Utils
 from ..helpers import getLogger
@@ -38,9 +38,7 @@ logSys = getLogger(__name__)
 #
 def asip(ip):
 	"""A little helper to guarantee ip being an IPAddr instance"""
-	if isinstance(ip, IPAddr):
-		return ip
-	return IPAddr(ip)
+	return ip if isinstance(ip, IPAddr) or ip is None else IPAddr(ip)
 
 
 ##
@@ -68,7 +66,7 @@ class DNSUtils:
 			ips = list()
 			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
 				ip = IPAddr(result[4][0])
-				if ip.isValidIP():
+				if ip.isValid:
 					ips.append(ip)
 		except socket.error, e:
 			# todo: make configurable the expired time of cache entry:
@@ -88,7 +86,7 @@ class DNSUtils:
 			if not isinstance(ip, IPAddr):
 				v = socket.gethostbyaddr(ip)[0]
 			else:
-				v = socket.gethostbyaddr(ip.ntoa())[0]
+				v = socket.gethostbyaddr(ip.ntoa)[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -104,7 +102,7 @@ class DNSUtils:
 		plainIP = IPAddr.searchIP(text)
 		if plainIP is not None:
 			ip = IPAddr(plainIP.group(0))
-			if ip.isValidIP():
+			if ip.isValid:
 				ipList.append(ip)
 
 		# If we are allowed to resolve -- give it a try if nothing was found
@@ -123,134 +121,132 @@ class DNSUtils:
 # Class for IP address handling.
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
-class IPAddr(object):
-	""" provide functions to handle IPv4 and IPv6 addresses 
+#
+class IPAddr:
+	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
 
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
 
 	# object attributes
-	addr = 0
-	family = socket.AF_UNSPEC
-	plen = 0
-	valid = False
-	raw = ""
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
-		ip = IPAddr.CACHE_OBJ.get(args)
-		if ip is not None:
-			return ip
-		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
-		return ip
+	_addr = 0
+	_family = socket.AF_UNSPEC
+	_plen = 0
+	_isValid = False
+	_raw = ""
 
 	# object methods
-	def __init(self, ipstring, cidr=-1):
+	def __init__(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
-				self.valid = True
-				break
 			except socket.error:
 				continue
+			else: 
+				self._isValid = True
+				break
 
-		if self.valid and family == socket.AF_INET:
+		if self.isValid and family == socket.AF_INET:
 			# convert host to network byte order
-			self.addr, = struct.unpack("!L", binary)
-			self.family = family
-			self.plen = 32
+			self._addr, = struct.unpack("!L", binary)
+			self._family = family
+			self._plen = 32
 
 			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
+			if cidr is not None and cidr >= 0:
 				mask = ~(0xFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
+				self._addr &= mask
+				self._plen = cidr
 
-		elif self.valid and family == socket.AF_INET6:
+		elif self.isValid and family == socket.AF_INET6:
 			# convert host to network byte order
 			hi, lo = struct.unpack("!QQ", binary)
-			self.addr = (hi << 64) | lo
-			self.family = family
-			self.plen = 128
+			self._addr = (hi << 64) | lo
+			self._family = family
+			self._plen = 128
 
 			# mask out host portion if prefix length is supplied
-			if cidr != None and cidr >= 0:
+			if cidr is not None and cidr >= 0:
 				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self.addr = self.addr & mask
-				self.plen = cidr
+				self._addr &= mask
+				self._plen = cidr
 
 			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr("::ffff:0:0", 96)):
-				self.addr = lo & 0xFFFFFFFFL
-				self.family = socket.AF_INET
-				self.plen = 32
+			elif self.isInNet(_IPv6_v4COMPAT):
+				self._addr = lo & 0xFFFFFFFFL
+				self._family = socket.AF_INET
+				self._plen = 32
 		else:
 			# string couldn't be converted neither to a IPv4 nor
 			# to a IPv6 address - retain raw input for later use
 			# (e.g. DNS resolution)
-			self.raw = ipstring
+			self._raw = ipstring
 
 	def __repr__(self):
-		return self.ntoa()
+		return self.ntoa
 
 	def __str__(self):
-		return self.ntoa()
+		return self.ntoa
+	
+	@property
+	def addr(self):
+		return self._addr
+
+	@property
+	def family(self):
+		return self._family
+
+	@property
+	def plen(self):
+		return self._plen
+
+	@property
+	def raw(self):
+		"""The raw address
+
+		Should only be set to a non-empty string if prior address
+		conversion wasn't possible
+		"""
+		return self._raw
+
+	@property
+	def isValid(self):
+		"""Either the object corresponds to a valid IP address
+		"""
+		return self._isValid
 
 	def __eq__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw == other.raw
-		if not self.valid or not other.valid: return False
-		if self.addr != other.addr: return False
-		if self.family != other.family: return False
-		if self.plen != other.plen: return False
-		return True
+		if not (self.isValid or other.isValid):
+			return self.raw == other.raw
+		return (
+			(self.isValid and other.isValid) and
+			(self.addr == other.addr) and
+			(self.family == other.family) and
+			(self.plen == other.plen)
+		)
 
 	def __ne__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return True
-			other = IPAddr(other)
-		if not self.valid and not other.valid: return self.raw != other.raw
-		if self.addr != other.addr: return True
-		if self.family != other.family: return True
-		if self.plen != other.plen: return True
-		return False
+		return not (self == other)
 
 	def __lt__(self, other):
-		if not isinstance(other, IPAddr):
-			if other is None: return False
-			other = IPAddr(other)
 		return self.family < other.family or self.addr < other.addr
 
 	def __add__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
-		if not isinstance(other, IPAddr):
-			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		# should be the same as by string (because of possible compare with string):
-		return hash(self.ntoa())
-		#return hash(self.addr)^hash((self.plen<<16)|self.family)
+		return hash(self.addr) ^ hash((self.plen << 16) | self.family)
 
+	@property
 	def hexdump(self):
-		""" dump the ip address in as a hex sequence in
-			network byte order - for debug purpose
+		"""Hex representation of the IP address (for debug purposes)
 		"""
 		if self.family == socket.AF_INET:
 			return "%08x" % self.addr
@@ -258,132 +254,94 @@ class IPAddr(object):
 			return "%032x" % self.addr
 		else:
 			return ""
-	
+
+	# TODO: could be lazily evaluated
+	@property
 	def ntoa(self):
-		""" represent IP object as text like the depricated 
-			C pendant inet_ntoa() but address family independent
+		""" represent IP object as text like the deprecated
+			C pendant inet.ntoa but address family independent
 		"""
-		if self.family == socket.AF_INET:
+		if self.isIPv4:
 			# convert network to host byte order
-			binary = struct.pack("!L", self.addr)
-		elif self.family == socket.AF_INET6:
+			binary = struct.pack("!L", self._addr)
+		elif self.isIPv6:
 			# convert network to host byte order
 			hi = self.addr >> 64
 			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
 		else:
-			return self.getRaw()
+			return self.raw
 
 		return socket.inet_ntop(self.family, binary)
 
 	def getPTR(self, suffix=""):
-		""" generates the DNS PTR string of the provided IP address object
-			if "suffix" is provided it will be appended as the second and top
+		""" return the DNS PTR string of the provided IP address object
+
+			If "suffix" is provided it will be appended as the second and top
 			level reverse domain.
-			if omitted it is implicitely set to the second and top level reverse
+			If omitted it is implicitly set to the second and top level reverse
 			domain of the according IP address family
 		"""
-		if self.family == socket.AF_INET:
-			reversed_ip = ".".join(reversed(self.ntoa().split(".")))
+		if self.isIPv4:
+			exploded_ip = self.ntoa.split(".")
 			if not suffix:
 				suffix = "in-addr.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-
-		elif self.family == socket.AF_INET6:
-			reversed_ip = ".".join(reversed(self.hexdump()))
+		elif self.isIPv6:
+			exploded_ip = self.hexdump()
 			if not suffix:
-				suffix =  "ip6.arpa."
-
-			return "%s.%s" % (reversed_ip, suffix)
-			
+				suffix = "ip6.arpa."
 		else:
 			return ""
 
+		return "%s.%s" % (".".join(reversed(exploded_ip)), suffix)
+
+	@property
 	def isIPv4(self):
-		""" return true if the IP object is of address family AF_INET
+		"""Either the IP object is of address family AF_INET
 		"""
 		return self.family == socket.AF_INET
 
+	@property
 	def isIPv6(self):
-		""" return true if the IP object is of address family AF_INET6
+		"""Either the IP object is of address family AF_INET6
 		"""
 		return self.family == socket.AF_INET6
 
-	def getRaw(self):
-		""" returns the raw attribute - should only be set
-			to a non-empty string if prior address conversion
-			wasn't possible
-		"""
-		return self.raw
-
-	def isValidIP(self):
-		""" returns true if the IP object has been created
-			from a valid IP address or false if not
-		"""
-		return self.valid
-
-	
 	def isInNet(self, net):
-		""" returns true if the IP object is in the provided
-			network (object)
+		"""Return either the IP object is in the provided network
 		"""
-		# if it isn't a valid IP address, try DNS resolution
-		if not net.isValidIP() and net.getRaw() != "":
-			# Check if IP in DNS
-			return self in DNSUtils.dnsToIp(net.getRaw())
-
 		if self.family != net.family:
 			return False
-
-		if self.family == socket.AF_INET:
+		if self.isIPv4:
 			mask = ~(0xFFFFFFFFL >> net.plen)
-
-		elif self.family == socket.AF_INET6:
+		elif self.isIPv6:
 			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
 		else:
 			return False
 		
-		if self.addr & mask == net.addr:
-			return True
+		return self.addr & mask == net.addr
 
-		return False
-
-	@property
-	def maskplen(self):
-		plen = 0
-		if (hasattr(self, '_maskplen')):
-			return self._plen
-		maddr = self.addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (self.ntoa(),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
-
-		
 	@staticmethod
-	def masktoplen(maskstr):
-		""" converts mask string to prefix length
-			only used for IPv4 masks
-		"""
-		return IPAddr(maskstr).maskplen
+	def masktoplen(mask):
+		"""Convert mask string to prefix length
 
+		To be used only for IPv4 masks
+		"""
+		mask = mask.addr  # to avoid side-effect within original mask
+		plen = 0
+		while mask:
+			mask = (mask << 1) & 0xFFFFFFFFL
+			plen += 1
+		return plen
 
 	@staticmethod
 	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
+		"""Search if text is an IP address, and return it if so, else None
 		"""
 		match = IPAddr.IP_CRE.match(text)
-		if match:
-			return match
-		else:
+		if not match:
 			match = IPAddr.IP6_CRE.match(text)
-			if match:
-				return match
-			else:
-				return None
+		return match if match else None
 
+# An IPv4 compatible IPv6 to be reused
+_IPv6_v4COMPAT = IPAddr("::ffff:0:0", 96)
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index d14030ea..704aedd0 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -107,5 +107,3 @@ class BeautifierTest(unittest.TestCase):
 
 		self.assertEqual(self.b.beautify(response), output)
 
-
-	

From 9b06c325e12c49de1ef54417c428471820399341 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 10:08:20 +0200
Subject: [PATCH 053/623] 2nd wave: code review, simplification, pythonization,
 etc. (test cases passed)

---
 fail2ban/server/ipdns.py                   | 110 ++++++++++++++-------
 fail2ban/tests/clientbeautifiertestcase.py |   2 +-
 2 files changed, 77 insertions(+), 35 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 744a8a73..da820366 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -38,7 +38,9 @@ logSys = getLogger(__name__)
 #
 def asip(ip):
 	"""A little helper to guarantee ip being an IPAddr instance"""
-	return ip if isinstance(ip, IPAddr) or ip is None else IPAddr(ip)
+	if isinstance(ip, IPAddr):
+		return ip
+	return IPAddr(ip)
 
 
 ##
@@ -122,38 +124,53 @@ class DNSUtils:
 #
 # This class contains methods for handling IPv4 and IPv6 addresses.
 #
-class IPAddr:
+class IPAddr(object):
 	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
 
 	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
 	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+	# An IPv4 compatible IPv6 to be reused (see below)
+	IP6_4COMPAT = None
 
 	# object attributes
-	_addr = 0
-	_family = socket.AF_UNSPEC
-	_plen = 0
-	_isValid = False
-	_raw = ""
+	__slots__ = '_family','_addr','_plen','_maskplen','_raw'
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstring, cidr=-1):
+		# already correct IPAddr
+		args = (ipstring, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstring, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
 
 	# object methods
-	def __init__(self, ipstring, cidr=-1):
+	def __init(self, ipstring, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
+		self._family = socket.AF_UNSPEC
+		self._addr = 0
+		self._plen = 0
+		self._maskplen = None
+		self._raw = ""
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstring)
+				self._family = family
+				break
 			except socket.error:
 				continue
-			else: 
-				self._isValid = True
-				break
 
-		if self.isValid and family == socket.AF_INET:
+		if self._family == socket.AF_INET:
 			# convert host to network byte order
 			self._addr, = struct.unpack("!L", binary)
-			self._family = family
 			self._plen = 32
 
 			# mask out host portion if prefix length is supplied
@@ -162,11 +179,10 @@ class IPAddr:
 				self._addr &= mask
 				self._plen = cidr
 
-		elif self.isValid and family == socket.AF_INET6:
+		elif self._family == socket.AF_INET6:
 			# convert host to network byte order
 			hi, lo = struct.unpack("!QQ", binary)
 			self._addr = (hi << 64) | lo
-			self._family = family
 			self._plen = 128
 
 			# mask out host portion if prefix length is supplied
@@ -176,7 +192,7 @@ class IPAddr:
 				self._plen = cidr
 
 			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(_IPv6_v4COMPAT):
+			elif self.isInNet(IPAddr.IP6_4COMPAT):
 				self._addr = lo & 0xFFFFFFFFL
 				self._family = socket.AF_INET
 				self._plen = 32
@@ -217,32 +233,43 @@ class IPAddr:
 	def isValid(self):
 		"""Either the object corresponds to a valid IP address
 		"""
-		return self._isValid
+		return self._family != socket.AF_UNSPEC
 
 	def __eq__(self, other):
-		if not (self.isValid or other.isValid):
-			return self.raw == other.raw
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		if self._family != other._family: return False
+		if self._family == socket.AF_UNSPEC:
+			return self._raw == other._raw
 		return (
-			(self.isValid and other.isValid) and
-			(self.addr == other.addr) and
-			(self.family == other.family) and
-			(self.plen == other.plen)
+			(self._addr == other._addr) and
+			(self._plen == other._plen)
 		)
 
 	def __ne__(self, other):
 		return not (self == other)
 
 	def __lt__(self, other):
-		return self.family < other.family or self.addr < other.addr
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		return self._family < other._family or self._addr < other._addr
 
 	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (self, other)
 
 	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
 		return "%s%s" % (other, self)
 
 	def __hash__(self):
-		return hash(self.addr) ^ hash((self.plen << 16) | self.family)
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa)
+		#return hash(self._addr)^hash((self._plen<<16)|self._family)
 
 	@property
 	def hexdump(self):
@@ -270,7 +297,7 @@ class IPAddr:
 			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
 		else:
-			return self.raw
+			return self._raw
 
 		return socket.inet_ntop(self.family, binary)
 
@@ -310,6 +337,11 @@ class IPAddr:
 	def isInNet(self, net):
 		"""Return either the IP object is in the provided network
 		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValid and net.raw != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.raw)
+
 		if self.family != net.family:
 			return False
 		if self.isIPv4:
@@ -319,20 +351,29 @@ class IPAddr:
 		else:
 			return False
 		
-		return self.addr & mask == net.addr
+		return (self.addr & mask) == net.addr
 
+	@property
+	def maskplen(self):
+		plen = 0
+		if self._maskplen is not None:
+			return self._plen
+		maddr = self.addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			plen += 1
+		self._maskplen = plen
+		return plen
+		
 	@staticmethod
 	def masktoplen(mask):
 		"""Convert mask string to prefix length
 
 		To be used only for IPv4 masks
 		"""
-		mask = mask.addr  # to avoid side-effect within original mask
-		plen = 0
-		while mask:
-			mask = (mask << 1) & 0xFFFFFFFFL
-			plen += 1
-		return plen
+		return IPAddr(mask).maskplen
 
 	@staticmethod
 	def searchIP(text):
@@ -343,5 +384,6 @@ class IPAddr:
 			match = IPAddr.IP6_CRE.match(text)
 		return match if match else None
 
+
 # An IPv4 compatible IPv6 to be reused
-_IPv6_v4COMPAT = IPAddr("::ffff:0:0", 96)
+IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96)
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 704aedd0..397bce1d 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -25,7 +25,7 @@ import unittest
 
 from ..client.beautifier import Beautifier
 from ..version import version
-from ..ipaddr import IPAddr
+from ..server.ipdns import IPAddr
 
 class BeautifierTest(unittest.TestCase):
 

From 25d6cf8dd23e150055baa67d49a122712fa83379 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 19:00:06 +0200
Subject: [PATCH 054/623] fix suhosin_log in common paths - log files should be
 separated using "\n": prevents to throw an error "File option must be 'head'
 or 'tail'", if jail suhosin will be enabled.

---
 config/paths-common.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index e2f08325..9072136c 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.
 
-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s
 
 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s

From 2497b05abc30a2232c6cd846b11bc04332b9ab03 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 May 2016 22:42:59 +0200
Subject: [PATCH 055/623] test cases preliminary extended;

---
 fail2ban/tests/servertestcase.py | 228 +++++++++++++++++++++++++++++++
 fail2ban/tests/utils.py          |   3 +
 2 files changed, 231 insertions(+)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d9db6c47..99afa012 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -33,7 +33,9 @@ import sys
 import platform
 
 from ..server.failregex import Regex, FailRegex, RegexException
+from ..server import actions as _actions
 from ..server.server import Server
+from ..server.ipdns import IPAddr
 from ..server.jail import Jail
 from ..server.jailthread import JailThread
 from ..server.utils import Utils
@@ -49,6 +51,8 @@ except ImportError: # pragma: no cover
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 FAST_BACKEND = "polling"
 
+logSys = getLogger("fail2ban")
+
 
 class TestServer(Server):
 	def setLogLevel(self, *args, **kwargs):
@@ -963,3 +967,227 @@ class LoggingTests(LogCaptureTestCase):
 			sys.__excepthook__ = prev_exchook
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
+
+
+from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK
+
+class ServerConfigReaderTests(LogCaptureTestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(ServerConfigReaderTests, self).__init__(*args, **kwargs)
+		self.__share_cfg = {}
+
+	def setUp(self):
+		"""Call before every test case."""
+		super(ServerConfigReaderTests, self).setUp()
+		self._execCmdLst = []
+	
+	def tearDown(self):
+		"""Call after every test case."""
+		super(ServerConfigReaderTests, self).tearDown()
+
+	def _executeCmd(self, realCmd, timeout=60):
+		for l in realCmd.split('\n'):
+			if not l.startswith('#'):
+				logSys.debug('exec-cmd: `%s`', l)
+			else:
+				logSys.debug(l)
+		return True
+
+	def test_IPAddr(self):
+		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
+		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
+
+	if STOCK:
+
+		def testCheckStockJailActions(self):
+			return
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			self.assertTrue(jails.read())		  # opens fine
+			self.assertTrue(jails.getOptions())	  # reads fine
+			stream = jails.convert(allow_no_files=True)
+
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			# for cmd in stream:
+			# 	print(cmd)
+
+			# filter all start commands (we want not start all jails):
+			for cmd in stream:
+				if cmd[0] != 'start':
+					# change to the fast init backend:
+					if cmd[0] == 'add':
+						cmd[2] = 'polling'
+					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
+					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
+						cmd[3] = "DUMMY-REGEX <HOST>"
+					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
+					try:
+						cmdHandler(cmd)
+					except Exception, e: # pragma: no cover
+						self.fail("Command %r has failed. Received %r" % (cmd, e))
+
+			# jails = server._Server__jails
+			# for j in jails:
+			# 	print(j, jails[j])
+
+		def getDefaultJailStream(self, jail, act):
+			act = act.replace('%(__name__)s', jail)
+			actName, actOpt = JailReader.extractOptions(act)
+			stream = [
+				['add', jail, 'polling'],
+				# ['set', jail, 'addfailregex', 'DUMMY-REGEX <HOST>'],
+			]
+			action = ActionReader(
+				actName, jail, actOpt,
+				share_config=self.__share_cfg, basedir=CONFIG_DIR)
+			self.assertTrue(action.read())
+			action.getOptions({})
+			stream.extend(action.convert())
+			return stream
+
+		def _assertLoggedAllTests(self, tests):
+			for t in tests:
+				self.assertLogged(t)
+
+
+		def testCheckStockCommandActions(self):
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			testJailsActions = (
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-mp`",
+						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -N f2b-j-w-iptables-mp`",
+						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`iptables -w -F f2b-j-w-iptables-mp`",
+						"`iptables -w -X f2b-j-w-iptables-mp`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -F f2b-j-w-iptables-mp`",
+						"`ip6tables -w -X f2b-j-w-iptables-mp`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-mp 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-mp -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-mp 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-mp -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-ap`",
+						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -N f2b-j-w-iptables-ap`",
+						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`iptables -w -F f2b-j-w-iptables-ap`",
+						"`iptables -w -X f2b-j-w-iptables-ap`",
+						"`ip6tables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -F f2b-j-w-iptables-ap`",
+						"`ip6tables -w -X f2b-j-w-iptables-ap`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-ap 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-ap -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-ap 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+			)
+
+			for jail, act, tests in testJailsActions:
+				stream = self.getDefaultJailStream(jail, act)
+
+				# for cmd in stream:
+				# 	print(cmd)
+
+				# filter all start commands (we want not start all jails):
+				for cmd in stream:
+					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
+					try:
+						cmdHandler(cmd)
+					except Exception, e: # pragma: no cover
+						self.fail("Command %r has failed. Received %r" % (cmd, e))
+
+			jails = server._Server__jails
+
+			for jail, act, tests in testJailsActions:
+				# print(jail, jails[jail])
+				for a in jails[jail].actions:
+					action = jails[jail].actions[a]
+					logSys.debug('# ' + ('=' * 50))
+					logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+					logSys.debug('# ' + ('=' * 50))
+					self.assertTrue(isinstance(action, _actions.CommandAction))
+					# wrap default command processor:
+					action.executeCmd = self._executeCmd
+					# test start :
+					logSys.debug('# === start ==='); self.pruneLog()
+					action.start()
+					self._assertLoggedAllTests(tests['start'])
+					# test ban ip4 :
+					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('192.0.2.1')})
+					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-ban'])
+					self.assertNotLogged(tests['ip6'])
+					# test unban ip4 :
+					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('192.0.2.1')})
+					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-unban'])
+					self.assertNotLogged(tests['ip6'])
+					# test ban ip6 :
+					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('2001:DB8::')})
+					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-ban'])
+					self.assertNotLogged(tests['ip4'])
+					# test unban ip6 :
+					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('2001:DB8::')})
+					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-unban'])
+					self.assertNotLogged(tests['ip4'])
+					# test stop :
+					logSys.debug('# === stop ==='); self.pruneLog()
+					action.stop()
+					self._assertLoggedAllTests(tests['stop'])
+
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 4c429105..fe3ab783 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -174,6 +174,7 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(servertestcase.JailTests))
 	tests.addTest(unittest.makeSuite(servertestcase.RegexTests))
 	tests.addTest(unittest.makeSuite(servertestcase.LoggingTests))
+	tests.addTest(unittest.makeSuite(servertestcase.ServerConfigReaderTests))
 	tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest))
 	tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions))
 	# Ticket, BanTicket, FailTicket
@@ -356,6 +357,8 @@ class LogCaptureTestCase(unittest.TestCase):
 				return
 		raise AssertionError("All of the %r were found present in the log: %r" % (s, logged))
 
+	def pruneLog(self):
+		self._log.truncate(0)
 
 	def getLog(self):
 		return self._log.getvalue()

From 43c0f3cdc43a31cfdf42b12d148085f3bce0dea0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 10:38:33 +0200
Subject: [PATCH 056/623] test cases extended

---
 fail2ban/tests/servertestcase.py | 60 +++++++++++++++++++++++---------
 fail2ban/tests/utils.py          | 23 ++++++++----
 2 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 99afa012..bac64dd1 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1001,7 +1001,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 	if STOCK:
 
 		def testCheckStockJailActions(self):
-			return
 			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
@@ -1020,6 +1019,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# change to the fast init backend:
 					if cmd[0] == 'add':
 						cmd[2] = 'polling'
+					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
+					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
+						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
 					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
 					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
@@ -1049,11 +1051,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			stream.extend(action.convert())
 			return stream
 
-		def _assertLoggedAllTests(self, tests):
-			for t in tests:
-				self.assertLogged(t)
-
-
 		def testCheckStockCommandActions(self):
 			server = TestServer()
 			transm = server._Server__transm
@@ -1134,6 +1131,37 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ' f2b-j-w-iptables-ipset ', 'ip6': ' f2b-j-w-iptables-ipset6 ',
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset`",
+						"`ipset destroy f2b-j-w-iptables-ipset`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset6`",
+						"`ipset destroy f2b-j-w-iptables-ipset6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`",
+					),					
+				}),
 			)
 
 			for jail, act, tests in testJailsActions:
@@ -1144,11 +1172,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 				# filter all start commands (we want not start all jails):
 				for cmd in stream:
-					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
-					try:
-						cmdHandler(cmd)
-					except Exception, e: # pragma: no cover
-						self.fail("Command %r has failed. Received %r" % (cmd, e))
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
 
 			jails = server._Server__jails
 
@@ -1165,29 +1191,29 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# test start :
 					logSys.debug('# === start ==='); self.pruneLog()
 					action.start()
-					self._assertLoggedAllTests(tests['start'])
+					self.assertLogged(*tests['start'], all=True)
 					# test ban ip4 :
 					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('192.0.2.1')})
-					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-ban'])
+					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
 					self.assertNotLogged(tests['ip6'])
 					# test unban ip4 :
 					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('192.0.2.1')})
-					self._assertLoggedAllTests(tests['ip4-check']+tests['ip4-unban'])
+					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
 					self.assertNotLogged(tests['ip6'])
 					# test ban ip6 :
 					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('2001:DB8::')})
-					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-ban'])
+					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
 					self.assertNotLogged(tests['ip4'])
 					# test unban ip6 :
 					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('2001:DB8::')})
-					self._assertLoggedAllTests(tests['ip6-check']+tests['ip6-unban'])
+					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(tests['ip4'])
 					# test stop :
 					logSys.debug('# === stop ==='); self.pruneLog()
 					action.stop()
-					self._assertLoggedAllTests(tests['stop'])
+					self.assertLogged(*tests['stop'], all=True)
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index fe3ab783..52097fa5 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -325,7 +325,7 @@ class LogCaptureTestCase(unittest.TestCase):
 	def _is_logged(self, s):
 		return s in self._log.getvalue()
 
-	def assertLogged(self, *s):
+	def assertLogged(self, *s, **kwargs):
 		"""Assert that one of the strings was logged
 
 		Preferable to assertTrue(self._is_logged(..)))
@@ -335,12 +335,22 @@ class LogCaptureTestCase(unittest.TestCase):
 		----------
 		s : string or list/set/tuple of strings
 		  Test should succeed if string (or any of the listed) is present in the log
+		all : boolean, should find all in s
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ in logged:
-				return
-		raise AssertionError("None among %r was found in the log: %r" % (s, logged))
+		if not kwargs.get('all', False):
+			# at least one entry should be found:
+			for s_ in s:
+				if s_ in logged:
+					return
+			# pragma: no cover
+			self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
+		else:
+			# each entry should be found:
+			for s_ in s:
+				if s_ not in logged:
+					# pragma: no cover
+					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
 	def assertNotLogged(self, *s):
 		"""Assert that strings were not logged
@@ -355,7 +365,8 @@ class LogCaptureTestCase(unittest.TestCase):
 		for s_ in s:
 			if s_ not in logged:
 				return
-		raise AssertionError("All of the %r were found present in the log: %r" % (s, logged))
+		# pragma: no cover
+		self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
 
 	def pruneLog(self):
 		self._log.truncate(0)

From 1a6450643d2d96738500d45a1f8ebe388f9f1b9f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 17:52:46 +0200
Subject: [PATCH 057/623] partially cherry pick from branch 'multi-set',
 prepare for conditional config parameters logic: - new readers logic (group
 some by multiple parameters 'set' -> 'multi-set'; - prevent to add 'known/'
 parameters twice (by merge section etc); - test cases fixed;

# Conflicts:
#	fail2ban/client/actionreader.py
---
 fail2ban/client/actionreader.py        | 38 +++++++++----------
 fail2ban/client/configparserinc.py     |  4 +-
 fail2ban/client/configreader.py        | 52 +++++++++++++++-----------
 fail2ban/client/fail2banregex.py       |  9 ++++-
 fail2ban/client/filterreader.py        | 22 +++++------
 fail2ban/client/jailreader.py          | 36 +++++++++---------
 fail2ban/server/server.py              | 18 +++++++--
 fail2ban/server/transmitter.py         | 37 +++++++++++++-----
 fail2ban/tests/clientreadertestcase.py | 42 +++++++++++++--------
 fail2ban/tests/samplestestcase.py      | 23 ++++++++----
 fail2ban/tests/servertestcase.py       | 19 ++++++----
 fail2ban/tests/utils.py                | 27 +++++++------
 12 files changed, 196 insertions(+), 131 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index c80b230e..698360ac 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -35,13 +35,13 @@ logSys = getLogger(__name__)
 
 class ActionReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "actionstart", None],
-		["string", "actionstop", None],
-		["string", "actioncheck", None],
-		["string", "actionban", None],
-		["string", "actionunban", None],
-	]
+	_configOpts = {
+		"actionstart": ["string", None],
+		"actionstop": ["string", None],
+		"actioncheck": ["string", None],
+		"actionban": ["string", None],
+		"actionunban": ["string", None],
+	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
 		self._name = initOpts.get("actname", file_)
@@ -65,20 +65,16 @@ class ActionReader(DefinitionInitConfigReader):
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
-		head.extend(["action", self._name])
-		for opt in self._opts:
-			if opt == "actionstart":
-				stream.append(head + ["actionstart", self._opts[opt]])
-			elif opt == "actionstop":
-				stream.append(head + ["actionstop", self._opts[opt]])
-			elif opt == "actioncheck":
-				stream.append(head + ["actioncheck", self._opts[opt]])
-			elif opt == "actionban":
-				stream.append(head + ["actionban", self._opts[opt]])
-			elif opt == "actionunban":
-				stream.append(head + ["actionunban", self._opts[opt]])
+		multi = []
+		for opt, optval in self._opts.iteritems():
+			if opt in self._configOpts:
+				multi.append([opt, optval])
 		if self._initOpts:
-			for p in self._initOpts:
-				stream.append(head + [p, self._initOpts[p]])
+			for opt, optval in self._initOpts.iteritems():
+				multi.append([opt, optval])
+		if len(multi) > 1:
+			stream.append(["multi-set", self._jailName, "action", self._name, multi])
+		elif len(multi):
+			stream.append(["set", self._jailName, "action", self._name] + multi[0])
 
 		return stream
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 7bbc7886..f4975857 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -231,7 +231,7 @@ after = 1.conf
 								# save previous known values, for possible using in local interpolations later:
 								sk = {}
 								for k, v in s2.iteritems():
-									if not k.startswith('known/'):
+									if not k.startswith('known/') and k != '__name__':
 										sk['known/'+k] = v
 								s2.update(sk)
 								# merge section
@@ -256,7 +256,7 @@ after = 1.conf
 		alls = self.get_sections()
 		sk = {}
 		for k, v in options.iteritems():
-			if pref == '' or not k.startswith(pref):
+			if pref == '' or (not k.startswith(pref) and k != '__name__'):
 				sk[pref+k] = v
 		alls[section].update(sk)
 
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index c6dd1b60..37e66249 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -203,40 +203,47 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	#
 	# Read the given option in the configuration file. Default values
 	# are used...
-	# Each optionValues entry is composed of an array with:
-	# 0 -> the type of the option
-	# 1 -> the name of the option
-	# 2 -> the default value for the option
+	# Each options entry is composed of an array with:
+	#  [[type, name, default], ...]
+	# Or it is a dict:
+	#  {name: [type, default], ...}
 	
 	def getOptions(self, sec, options, pOptions=None):
 		values = dict()
-		for option in options:
-			try:
-				if option[0] == "bool":
-					v = self.getboolean(sec, option[1])
-				elif option[0] == "int":
-					v = self.getint(sec, option[1])
+		for optname in options:
+			if isinstance(options, (list,tuple)):
+				if len(optname) > 2:
+					opttype, optname, optvalue = optname
 				else:
-					v = self.get(sec, option[1])
-				if not pOptions is None and option[1] in pOptions:
+					(opttype, optname), optvalue = optname, None
+			else:
+				opttype, optvalue = options[optname]
+			try:
+				if opttype == "bool":
+					v = self.getboolean(sec, optname)
+				elif opttype == "int":
+					v = self.getint(sec, optname)
+				else:
+					v = self.get(sec, optname)
+				if not pOptions is None and optname in pOptions:
 					continue
-				values[option[1]] = v
+				values[optname] = v
 			except NoSectionError, e:
 				# No "Definition" section or wrong basedir
 				logSys.error(e)
-				values[option[1]] = option[2]
+				values[optname] = optvalue
 				# TODO: validate error handling here.
 			except NoOptionError:
-				if not option[2] is None:
+				if not optvalue is None:
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
-								% (option[1], sec, option[2]))
-					values[option[1]] = option[2]
+								% (optname, sec, optvalue))
+					values[optname] = optvalue
 				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec)
+					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
 			except ValueError:
-				logSys.warning("Wrong value for '" + option[1] + "' in '" + sec +
-							"'. Using default one: '" + repr(option[2]) + "'")
-				values[option[1]] = option[2]
+				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
+							"'. Using default one: '" + repr(optvalue) + "'")
+				values[optname] = optvalue
 		return values
 
 
@@ -286,7 +293,8 @@ class DefinitionInitConfigReader(ConfigReader):
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
 				v = self.get("Init", opt)
-				self._initOpts['known/'+opt] = v
+				if not opt.startswith('known/') and opt != '__name__':
+					self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
 	
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index f1a9c887..9ecb7229 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -291,7 +291,14 @@ class Fail2banRegex(object):
 				RegexStat(m[3])
 				for m in filter(
 					lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype,
-					readercommands)]
+					readercommands)
+			] + [
+				RegexStat(m)
+				for mm in filter(
+					lambda x: x[0] == 'multi-set' and x[2] == "add%sregex" % regextype,
+					readercommands)
+				for m in mm[3]
+			]
 			# Read out and set possible value of maxlines
 			for command in readercommands:
 				if command[2] == "maxlines":
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index 318c8c9a..8b30f914 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -37,10 +37,10 @@ logSys = getLogger(__name__)
 
 class FilterReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "ignoreregex", None],
-		["string", "failregex", ""],
-	]
+	_configOpts = {
+		"ignoreregex": ["string", None],
+		"failregex": ["string", ""],
+	}
 
 	def setFile(self, fileName):
 		self.__file = fileName
@@ -64,16 +64,16 @@ class FilterReader(DefinitionInitConfigReader):
 		if not len(opts):
 			return stream
 		for opt, value in opts.iteritems():
-			if opt == "failregex":
+			if opt in ("failregex", "ignoreregex"):
+				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self._jailName, "addfailregex", regex])
-			elif opt == "ignoreregex":
-				for regex in value.split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self._jailName, "addignoreregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self._jailName, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self._jailName, "add" + opt, multi[0]])
 		if self._initOpts:
 			if 'maxlines' in self._initOpts:
 				# We warn when multiline regex is used without maxlines > 1
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 327ddf1b..fda5d40c 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -190,11 +190,11 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
-		for opt in self.__opts:
+		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					self.__opts.get('backend', None) != "systemd":
 				found_files = 0
-				for path in self.__opts[opt].split("\n"):
+				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
 					path, tail = path if len(path) > 1 else (path[0], "head")
 					pathList = JailReader._glob(path)
@@ -208,32 +208,32 @@ class JailReader(ConfigReader):
 					raise ValueError(
 						"Have not found any log file for %s jail" % self.__name)
 			elif opt == "logencoding":
-				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
+				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
-				backend = self.__opts[opt]
+				backend = value
 			elif opt == "maxretry":
-				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
+				stream.append(["set", self.__name, "maxretry", value])
 			elif opt == "ignoreip":
-				for ip in splitcommaspace(self.__opts[opt]):
+				for ip in splitcommaspace(value):
 					stream.append(["set", self.__name, "addignoreip", ip])
 			elif opt == "findtime":
-				stream.append(["set", self.__name, "findtime", self.__opts[opt]])
+				stream.append(["set", self.__name, "findtime", value])
 			elif opt == "bantime":
-				stream.append(["set", self.__name, "bantime", self.__opts[opt]])
+				stream.append(["set", self.__name, "bantime", value])
 			elif opt == "usedns":
-				stream.append(["set", self.__name, "usedns", self.__opts[opt]])
-			elif opt == "failregex":
-				for regex in self.__opts[opt].split('\n'):
+				stream.append(["set", self.__name, "usedns", value])
+			elif opt in ("failregex", "ignoreregex"):
+				multi = []
+				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self.__name, "addfailregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self.__name, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self.__name, "add" + opt, multi[0]])
 			elif opt == "ignorecommand":
-				stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]])
-			elif opt == "ignoreregex":
-				for regex in self.__opts[opt].split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self.__name, "addignoreregex", regex])
+				stream.append(["set", self.__name, "ignorecommand", value])
 		if self.__filter:
 			stream.extend(self.__filter.convert())
 		for action in self.__actions:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 3bdfd71b..7f75c347 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -262,8 +262,13 @@ class Server:
 	def getIgnoreCommand(self, name):
 		return self.__jails[name].filter.getIgnoreCommand()
 
-	def addFailRegex(self, name, value):
-		self.__jails[name].filter.addFailRegex(value)
+	def addFailRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addFailRegex(value)
+		else:
+			flt.addFailRegex(value)
 	
 	def delFailRegex(self, name, index):
 		self.__jails[name].filter.delFailRegex(index)
@@ -271,8 +276,13 @@ class Server:
 	def getFailRegex(self, name):
 		return self.__jails[name].filter.getFailRegex()
 	
-	def addIgnoreRegex(self, name, value):
-		self.__jails[name].filter.addIgnoreRegex(value)
+	def addIgnoreRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addIgnoreRegex(value)
+		else:
+			flt.addIgnoreRegex(value)
 	
 	def delIgnoreRegex(self, name, index):
 		self.__jails[name].filter.delIgnoreRegex(index)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 4c4c32f7..29d6d189 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -99,6 +99,8 @@ class Transmitter:
 			return None
 		elif command[0] == "flushlogs":
 			return self.__server.flushLogs()
+		elif command[0] == "multi-set":
+			return self.__commandSet(command[1:], True)
 		elif command[0] == "set":
 			return self.__commandSet(command[1:])
 		elif command[0] == "get":
@@ -109,7 +111,7 @@ class Transmitter:
 			return version.version
 		raise Exception("Invalid command")
 	
-	def __commandSet(self, command):
+	def __commandSet(self, command, multiple=False):
 		name = command[0]
 		# Logging
 		if name == "loglevel":
@@ -196,7 +198,9 @@ class Transmitter:
 			return self.__server.getJournalMatch(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
-			self.__server.addFailRegex(name, value)
+			self.__server.addFailRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getFailRegex(name)
 		elif command[1] == "delfailregex":
 			value = int(command[2])
@@ -204,7 +208,9 @@ class Transmitter:
 			return self.__server.getFailRegex(name)
 		elif command[1] == "addignoreregex":
 			value = command[2]
-			self.__server.addIgnoreRegex(name, value)
+			self.__server.addIgnoreRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getIgnoreRegex(name)
 		elif command[1] == "delignoreregex":
 			value = int(command[2])
@@ -254,15 +260,26 @@ class Transmitter:
 			return None
 		elif command[1] == "action":
 			actionname = command[2]
-			actionkey = command[3]
 			action = self.__server.getAction(name, actionname)
-			if callable(getattr(action, actionkey, None)):
-				actionvalue = json.loads(command[4]) if len(command)>4 else {}
-				return getattr(action, actionkey)(**actionvalue)
+			if multiple:
+				for cmd in command[3]:
+					actionkey = cmd[0]
+					if callable(getattr(action, actionkey, None)):
+						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}
+						getattr(action, actionkey)(**actionvalue)
+					else:
+						actionvalue = cmd[1]
+						setattr(action, actionkey, actionvalue)
+				return True
 			else:
-				actionvalue = command[4]
-				setattr(action, actionkey, actionvalue)
-				return getattr(action, actionkey)
+				actionkey = command[3]
+				if callable(getattr(action, actionkey, None)):
+					actionvalue = json.loads(command[4]) if len(command)>4 else {}
+					return getattr(action, actionkey)(**actionvalue)
+				else:
+					actionvalue = command[4]
+					setattr(action, actionkey, actionvalue)
+					return getattr(action, actionkey)
 		raise Exception("Invalid command (no set action or not yet implemented)")
 	
 	def __commandGet(self, command):
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index bd734c1b..0edbc69e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -275,7 +275,15 @@ class JailReaderTest(LogCaptureTestCase):
 		# convert and get stream
 		stream = jail.convert()
 		# get action and retrieve agent from it, compare with agent saved in version:
-		act = [o for o in stream if len(o) > 4 and (o[4] == 'agent' or o[4].endswith('badips.py'))]
+		act = []
+		for cmd in stream:
+			if len(cmd) <= 4:
+				continue
+			# differentiate between set and multi-set (wrop it here to single set):
+			if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')):
+				act.append(cmd)
+			elif cmd[0] == 'multi-set':
+				act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent'])
 		useragent = 'Fail2Ban/%s' % version
 		self.assertEqual(len(act), 4)
 		self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent])
@@ -311,23 +319,21 @@ class FilterReaderTest(unittest.TestCase):
 		self.__share_cfg = {}
 
 	def testConvert(self):
-		output = [['set', 'testcase01', 'addfailregex',
+		output = [['multi-set', 'testcase01', 'addfailregex', [
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
-			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the underlying authentication mo"
-			"dule for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"dule for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the\\nunderlying authentication."
-			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"],
+			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"]],
 			['set', 'testcase01', 'addignoreregex', 
 			"^.+ john from host 192.168.1.1\\s*$"],
 			['set', 'testcase01', 'addjournalmatch',
@@ -495,9 +501,11 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertEqual(sorted(comm_commands),
 			sorted([['add', 'emptyaction', 'auto'],
 			 ['add', 'test-known-interp', 'auto'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 3 (jail.local) <HOST>'],
+			 ['multi-set', 'test-known-interp', 'addfailregex', [
+			   'failure test 1 (filter.d/test.conf) <HOST>',
+			   'failure test 2 (filter.d/test.local) <HOST>',
+			   'failure test 3 (jail.local) <HOST>'
+			 ]],
 			 ['start', 'test-known-interp'],
 			 ['add', 'missinglogfiles', 'auto'],
 			 ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
@@ -660,12 +668,16 @@ class JailsReaderTest(LogCaptureTestCase):
 						self.assertTrue('blocktype' in action._initOpts)
 						# Verify that we have a call to set it up
 						blocktype_present = False
-						target_command = ['set', jail_name, 'action', action_name, 'blocktype']
+						target_command = [jail_name, 'action', action_name]
 						for command in commands:
-							if (len(command) > 5 and
-								command[:5] == target_command):
-								blocktype_present = True
-								continue
+							if (len(command) > 4 and command[0] == 'multi-set' and
+								command[1:4] == target_command):
+									blocktype_present = ('blocktype' in [cmd[0] for cmd in command[4]])
+							elif (len(command) > 5 and command[0] == 'set' and
+								command[1:4] == target_command and command[4] == 'blocktype'): # pragma: no cover - because of multi-set
+									blocktype_present = True
+							if blocktype_present:
+								break
 						self.assertTrue(
 							blocktype_present,
 							msg="Found no %s command among %s"
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 2ed77554..074ba24c 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -72,14 +72,21 @@ def testSampleRegexsFactory(name):
 		filterConf.getOptions({})
 
 		for opt in filterConf.convert():
-			if opt[2] == "addfailregex":
-				self.filter.addFailRegex(opt[3])
-			elif opt[2] == "maxlines":
-				self.filter.setMaxLines(opt[3])
-			elif opt[2] == "addignoreregex":
-				self.filter.addIgnoreRegex(opt[3])
-			elif opt[2] == "datepattern":
-				self.filter.setDatePattern(opt[3])
+			if opt[0] == 'multi-set':
+				optval = opt[3]
+			elif opt[0] == 'set':
+				optval = [opt[3]]
+			else:
+				continue
+			for optval in optval:
+				if opt[2] == "addfailregex":
+					self.filter.addFailRegex(optval)
+				elif opt[2] == "addignoreregex":
+					self.filter.addIgnoreRegex(optval)
+				elif opt[2] == "maxlines":
+					self.filter.setMaxLines(optval)
+				elif opt[2] == "datepattern":
+					self.filter.setDatePattern(optval)
 
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index bac64dd1..0e109a01 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1024,7 +1024,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
 					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
-					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addfailregex':
+					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and (
+						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
+					):
+						cmd[0] = "set"
 						cmd[3] = "DUMMY-REGEX <HOST>"
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
 					try:
@@ -1058,7 +1061,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 			testJailsActions = (
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
-					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
 						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
@@ -1095,7 +1098,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
-					'ip4': '`iptables ', 'ip6': '`ip6tables ',
+					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
 						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
@@ -1132,7 +1135,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
-					'ip4': ' f2b-j-w-iptables-ipset ', 'ip6': ' f2b-j-w-iptables-ipset6 ',
+					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
 						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
@@ -1196,22 +1199,22 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
-					self.assertNotLogged(tests['ip6'])
+					self.assertNotLogged(*tests['ip6'], all=True)
 					# test unban ip4 :
 					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
-					self.assertNotLogged(tests['ip6'])
+					self.assertNotLogged(*tests['ip6'], all=True)
 					# test ban ip6 :
 					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
 					action.ban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
-					self.assertNotLogged(tests['ip4'])
+					self.assertNotLogged(*tests['ip4'], all=True)
 					# test unban ip6 :
 					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
 					action.unban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
-					self.assertNotLogged(tests['ip4'])
+					self.assertNotLogged(*tests['ip4'], all=True)
 					# test stop :
 					logSys.debug('# === stop ==='); self.pruneLog()
 					action.stop()
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 52097fa5..e97daebf 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -335,7 +335,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		----------
 		s : string or list/set/tuple of strings
 		  Test should succeed if string (or any of the listed) is present in the log
-		all : boolean, should find all in s
+		all : boolean (default False) if True should fail if any of s not logged
 		"""
 		logged = self._log.getvalue()
 		if not kwargs.get('all', False):
@@ -343,16 +343,15 @@ class LogCaptureTestCase(unittest.TestCase):
 			for s_ in s:
 				if s_ in logged:
 					return
-			# pragma: no cover
-			self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
+			if True: # pragma: no cover
+				self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
 		else:
 			# each entry should be found:
 			for s_ in s:
-				if s_ not in logged:
-					# pragma: no cover
+				if s_ not in logged: # pragma: no cover
 					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
-	def assertNotLogged(self, *s):
+	def assertNotLogged(self, *s, **kwargs):
 		"""Assert that strings were not logged
 
 		Parameters
@@ -360,13 +359,19 @@ class LogCaptureTestCase(unittest.TestCase):
 		s : string or list/set/tuple of strings
 		  Test should succeed if the string (or at least one of the listed) is not
 		  present in the log
+		all : boolean (default False) if True should fail if any of s logged
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ not in logged:
-				return
-		# pragma: no cover
-		self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
+		if not kwargs.get('all', False):
+			for s_ in s:
+				if s_ not in logged:
+					return
+			if True: # pragma: no cover
+				self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
+		else:
+			for s_ in s:
+				if s_ in logged: # pragma: no cover
+					self.fail("%r was found in the log: ===\n%s===" % (s_, logged))
 
 	def pruneLog(self):
 		self._log.truncate(0)

From ed2f3ef77d4c0443790bd0e99c4f54f1cc7668fc Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Mon, 14 Mar 2016 20:30:19 +0100
Subject: [PATCH 058/623] improve PF action and make IPv6 aware

---
 config/action.d/pf.conf | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index edcaa175..62296458 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -3,6 +3,7 @@
 # OpenBSD pf ban/unban
 #
 # Author: Nick Hilliard <nick@foobar.org>
+# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
 #
 #
 
@@ -12,23 +13,27 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-# we don't enable PF automatically, as it will be enabled elsewhere
-actionstart = 
+# we don't enable PF automatically; to enable run pfctl -e 
+# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to any port <port>" | pfctl -f-
 
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-# we don't disable PF automatically either
-actionstop = 
+# we only disable PF rules we've installed prior
+actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
+             pfctl -t <tablename>-<name> -T flush
+             pfctl -t <tablename>-<name> -T kill
 
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = 
+actioncheck = pfctl -sr | grep -q '<tablename>-<name>'
 
 
 # Option:  actionban
@@ -39,7 +44,7 @@ actioncheck =
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
+actionban = pfctl -t <tablename>-<name> -T add <ip>
 
 
 # Option:  actionunban
@@ -51,7 +56,7 @@ actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
 # Values:  CMD
 #
 # note -r option used to remove matching rule
-actionunban = /sbin/pfctl -t <tablename> -T delete <ip>/32
+actionunban = pfctl -t <tablename>-<name> -T delete <ip>
 
 [Init]
 # Option:  tablename
@@ -60,3 +65,15 @@ actionunban = /sbin/pfctl -t <tablename> -T delete <ip>/32
 #
 tablename = fail2ban
 
+# Option:  protocol
+# Notes.:  internally used by config reader for interpolations.
+# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
+#
+protocol = tcp
+
+# Option:  port
+# Notes.:  the port to block, defaults to any
+# Values:  [ STRING ]
+#
+port = any
+

From 75028585c02c9767d11c2deca440370aaef62214 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 3 May 2016 19:26:27 +0200
Subject: [PATCH 059/623] test cases extended for verifying ipv4/ipv6,
 normalized pf-action with test case

---
 config/action.d/pf.conf          |   6 +-
 fail2ban/tests/servertestcase.py | 179 +++++++++++++++++++++++++++++--
 2 files changed, 176 insertions(+), 9 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 62296458..effd080c 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -24,7 +24,7 @@ actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f-
 # Values:  CMD
 #
 # we only disable PF rules we've installed prior
-actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
+actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
              pfctl -t <tablename>-<name> -T flush
              pfctl -t <tablename>-<name> -T kill
 
@@ -33,7 +33,7 @@ actionstop = pfctl -sr 2>/dev/null | grep -v '<tablename>-<name>' | pfctl -f-
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = pfctl -sr | grep -q '<tablename>-<name>'
+actioncheck = pfctl -sr | grep -q <tablename>-<name>
 
 
 # Option:  actionban
@@ -63,7 +63,7 @@ actionunban = pfctl -t <tablename>-<name> -T delete <ip>
 # Notes.:  The pf table name.
 # Values:  [ STRING ]
 #
-tablename = fail2ban
+tablename = f2b
 
 # Option:  protocol
 # Notes.:  internally used by config reader for interpolations.
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 0e109a01..893976be 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1055,13 +1055,16 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			return stream
 
 		def testCheckStockCommandActions(self):
-			server = TestServer()
-			transm = server._Server__transm
-			cmdHandler = transm._Transmitter__commandHandler
-
+			# test cases to check valid ipv4/ipv6 action definition, tuple with (('jail', 'action[params]', 'tests', ...)
+			# where tests is a dictionary contains:
+			#   'ip4' - should not be found (logged) on ban/unban of IPv6 (negative test),
+			#   'ip6' - should not be found (logged) on ban/unban of IPv4 (negative test),
+			#   'start', 'stop' - should be found (logged) on action start/stop,
+			#   etc.
 			testJailsActions = (
+				# iptables-multiport --
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
-					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
 						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
@@ -1097,8 +1100,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-mp -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				# iptables-allports --
 				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
-					'ip4': ('`iptables ',), 'ip6': ('`ip6tables ',),
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
 						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
@@ -1134,6 +1138,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
+				# iptables-ipset-proto6 --
 				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
@@ -1165,7 +1170,169 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`",
 					),					
 				}),
+				# iptables-ipset-proto6-allports --
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
+						"`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap`",
+						"`ip6tables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap6`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`",
+					),					
+				}),
+				# iptables --
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables`",
+						"`iptables -w -A f2b-j-w-iptables -j RETURN`",
+						"`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -N f2b-j-w-iptables`",
+						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`iptables -w -F f2b-j-w-iptables`",
+						"`iptables -w -X f2b-j-w-iptables`",
+						"`ip6tables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -F f2b-j-w-iptables`",
+						"`ip6tables -w -X f2b-j-w-iptables`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-new --
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-new`",
+						"`iptables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -N f2b-j-w-iptables-new`",
+						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`iptables -w -F f2b-j-w-iptables-new`",
+						"`iptables -w -X f2b-j-w-iptables-new`",
+						"`ip6tables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -F f2b-j-w-iptables-new`",
+						"`ip6tables -w -X f2b-j-w-iptables-new`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-new 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-new -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-new 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-new -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-xt_recent-echo --
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
+					'start': (
+						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'stop': (
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+						"`if [ `id -u` -eq 0 ];then iptables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'ip4-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip4-ban': (
+						r"`echo +192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip4-unban': (
+						r"`echo -192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-ban': (
+						r"`echo +2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip6-unban': (
+						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),					
+				}),
+				# pf --
+				('j-w-pf', 'pf[name=%(__name__)s]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
+ 						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf -T flush`',
+						'`pfctl -t f2b-j-w-pf -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
+				}),
 			)
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
 
 			for jail, act, tests in testJailsActions:
 				stream = self.getDefaultJailStream(jail, act)

From 504e5ba6f22e9e0bc5121787e1ef080d2ca98c83 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 18:40:07 +0200
Subject: [PATCH 060/623] actions support IPv6 now:   - introduced
 "conditional" sections, see for example `[Init?family=inet6]`;   -
 iptables-common and other iptables config(s) made IPv6 capable;   - several
 small code optimizations; * all test cases passed (py3.x compatible);

---
 config/action.d/iptables-common.conf          | 20 +++++
 .../iptables-ipset-proto6-allports.conf       | 26 ++++--
 config/action.d/iptables-ipset-proto6.conf    | 26 ++++--
 config/action.d/iptables-xt_recent-echo.conf  | 24 +++--
 fail2ban/client/configparserinc.py            | 46 ++++++----
 fail2ban/server/action.py                     | 88 +++++++++++++------
 fail2ban/tests/actiontestcase.py              | 16 ++--
 fail2ban/tests/filtertestcase.py              |  2 -
 8 files changed, 176 insertions(+), 72 deletions(-)

diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf
index 45819e3d..a3921021 100644
--- a/config/action.d/iptables-common.conf
+++ b/config/action.d/iptables-common.conf
@@ -6,6 +6,9 @@
 # used in all iptables based actions by default.
 #
 # The user can override the defaults in iptables-common.local
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -13,6 +16,7 @@ after = iptables-blocktype.local
         iptables-common.local
 # iptables-blocktype.local is obsolete
 
+
 [Init]
 
 # Option:  chain
@@ -62,3 +66,19 @@ lockingopt = -w
 # Notes.:  Actual command to be executed, including common to all calls options
 # Values:  STRING
 iptables = iptables <lockingopt>
+
+
+[Init?family=inet6]
+
+# Option:  blocktype (ipv6)
+# Note:    This is what the action does with rules. This can be any jump target
+#          as per the iptables man page (section 8). Common values are DROP
+#          REJECT, REJECT --reject-with icmp6-port-unreachable
+# Values:  STRING
+blocktype = REJECT --reject-with icmp6-port-unreachable
+
+# Option:  iptables (ipv6)
+# Notes.:  Actual command to be executed, including common to all calls options
+# Values:  STRING
+iptables = ip6tables <lockingopt>
+
diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf
index 1f1d336f..113f599e 100644
--- a/config/action.d/iptables-ipset-proto6-allports.conf
+++ b/config/action.d/iptables-ipset-proto6-allports.conf
@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules which probably won't be protocol version 6.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blockty
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6
diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf
index 3b51ef58..dee7b029 100644
--- a/config/action.d/iptables-ipset-proto6.conf
+++ b/config/action.d/iptables-ipset-proto6.conf
@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6
diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf
index d3d43f86..018d2cf6 100644
--- a/config/action.d/iptables-xt_recent-echo.conf
+++ b/config/action.d/iptables-xt_recent-echo.conf
@@ -2,7 +2,8 @@
 #
 # Author: Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
 #
-#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable
 
 [INCLUDES]
 
@@ -22,30 +23,30 @@ before = iptables-common.conf
 # iptables-persistent package).
 # 
 # Explanation of the rule below:
-#    Check if any packets coming from an IP on the f2b-<name>
+#    Check if any packets coming from an IP on the <iptname>
 #    list have been seen in the last 3600 seconds. If yes, update the
 #    timestamp for this IP and drop the packet. If not, let the packet
 #    through.
 #
-#    Fail2ban inserts blacklisted hosts into the f2b-<name> list
+#    Fail2ban inserts blacklisted hosts into the <iptname> list
 #    and removes them from the list after some time, according to its
 #    own rules. The 3600 second timeout is independent and acts as a
 #    safeguard in case the fail2ban process dies unexpectedly. The
 #    shorter of the two timeouts actually matters.
-actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = echo / > /proc/net/xt_recent/f2b-<name>
-             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstop = echo / > /proc/net/xt_recent/<iptname>
+             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = test -e /proc/net/xt_recent/f2b-<name>
+actioncheck = test -e /proc/net/xt_recent/<iptname>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -53,7 +54,7 @@ actioncheck = test -e /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
+actionban = echo +<ip> > /proc/net/xt_recent/<iptname>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -61,7 +62,12 @@ actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = echo -<ip> > /proc/net/xt_recent/f2b-<name>
+actionunban = echo -<ip> > /proc/net/xt_recent/<iptname>
 
 [Init]
 
+iptname              = f2b-<name>
+
+[Init?family=inet6]
+
+iptname = f2b-<name>6
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index f4975857..617ff8f6 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -25,6 +25,7 @@ __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko'
 __license__ = 'GPL'
 
 import os
+import re
 import sys
 from ..helpers import getLogger
 
@@ -99,6 +100,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
+
 	if sys.version_info >= (3,2):
 		# overload constructor only for fancy new Python3's
 		def __init__(self, share_config=None, *args, **kwargs):
@@ -225,21 +228,31 @@ after = 1.conf
 					# merge defaults and all sections to self:
 					alld.update(cfg.get_defaults())
 					for n, s in cfg.get_sections().iteritems():
-						if isinstance(s, dict):
-							s2 = alls.get(n)
-							if isinstance(s2, dict):
-								# save previous known values, for possible using in local interpolations later:
-								sk = {}
-								for k, v in s2.iteritems():
-									if not k.startswith('known/') and k != '__name__':
-										sk['known/'+k] = v
-								s2.update(sk)
-								# merge section
-								s2.update(s)
-							else:
-								alls[n] = s.copy()
+						curalls = alls
+						# conditional sections
+						cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+						if cond:
+							n, cond = cond.groups()
+							s = s.copy()
+							try: 
+								del(s['__name__'])
+							except KeyError:
+								pass
+							for k in s.keys():
+								v = s.pop(k)
+								s[k + cond] = v
+						s2 = alls.get(n)
+						if isinstance(s2, dict):
+							# save previous known values, for possible using in local interpolations later:
+							sk = {}
+							for k, v in s2.iteritems():
+								if not k.startswith('known/') and k != '__name__':
+									sk['known/'+k] = v
+							s2.update(sk)
+							# merge section
+							s2.update(s)
 						else:
-							alls[n] = s
+							alls[n] = s.copy()
 
 			return ret
 
@@ -254,9 +267,12 @@ after = 1.conf
 
 	def merge_section(self, section, options, pref='known/'):
 		alls = self.get_sections()
+		if pref == '':
+			alls[section].update(options)
+			return
 		sk = {}
 		for k, v in options.iteritems():
-			if pref == '' or (not k.startswith(pref) and k != '__name__'):
+			if not k.startswith(pref) and k != '__name__':
 				sk[pref+k] = v
 		alls[section].update(sk)
 
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 446ae853..cfa03403 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -32,6 +32,7 @@ import time
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .ipdns import asip
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -41,6 +42,11 @@ logSys = getLogger(__name__)
 # Create a lock for running system commands
 _cmd_lock = threading.Lock()
 
+# Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
+allowed_ipv6 = True
+
+# compiled RE for tag name (replacement name) 
+TAG_CRE = re.compile(r'<([^ <>]+)>')
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -256,14 +262,18 @@ class CommandAction(ActionBase):
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		if (self._properties and
-			not self.substituteRecursiveTags(self._properties)):
-			self._logSys.error(
-				"properties contain self referencing definitions "
-				"and cannot be resolved")
-			raise RuntimeError("Error starting action")
-		startCmd = self.replaceTag(self.actionstart, self._properties)
-		if not self.executeCmd(startCmd, self.timeout):
+		# check valid tags in properties (raises ValueError if self recursion, etc.):
+		if self._properties:
+			self.substituteRecursiveTags(self._properties)
+		# common (resp. ipv4):
+		startCmd = self.replaceTag('<actionstart>', self._properties, conditional='family=inet4')
+		res = self.executeCmd(startCmd, self.timeout)
+		# start ipv6 actions if available:
+		if allowed_ipv6:
+			startCmd6 = self.replaceTag('<actionstart>', self._properties, conditional='family=inet6')
+			if startCmd6 != startCmd:
+				res &= self.executeCmd(startCmd6, self.timeout)
+		if not res:
 			raise RuntimeError("Error starting action")
 
 	@property
@@ -289,7 +299,7 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionban, aInfo):
+		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
 	@property
@@ -315,7 +325,7 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionunban, aInfo):
+		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
 	@property
@@ -350,12 +360,19 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		stopCmd = self.replaceTag(self.actionstop, self._properties)
-		if not self.executeCmd(stopCmd, self.timeout):
+		# common (resp. ipv4):
+		stopCmd = self.replaceTag('<actionstop>', self._properties, conditional='family=inet4')
+		res = self.executeCmd(stopCmd, self.timeout)
+		# ipv6 actions if available:
+		if allowed_ipv6:
+			stopCmd6 = self.replaceTag('<actionstop>', self._properties, conditional='family=inet6')
+			if stopCmd6 != stopCmd:
+				res &= self.executeCmd(stopCmd6, self.timeout)
+		if not res:
 			raise RuntimeError("Error stopping action")
 
 	@classmethod
-	def substituteRecursiveTags(cls, tags):
+	def substituteRecursiveTags(cls, tags, conditional=''):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -374,7 +391,7 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
-		t = re.compile(r'<([^ <>]+)>')
+		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
 		while True:
 			repFlag = False
@@ -394,14 +411,21 @@ class CommandAction(ActionBase):
 					if found_tag == tag or found_tag in done:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
-						return False
-					if found_tag in cls._escapedTags or not found_tag in tags:
+						raise ValueError(
+							"properties contain self referencing definitions "
+							"and cannot be resolved, fail tag: %s value: %s" % (tag, value))
+					repl = None
+					if found_tag not in cls._escapedTags:
+						repl = tags.get(found_tag + '?' + conditional)
+						if repl is None:
+							repl = tags.get(found_tag)
+					if repl is None:
 						# Escaped or missing tags - just continue on searching after end of match
 						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
 						# constructs like <STDIN>.
 						m = t.search(value, m.end())
 						continue
-					value = value.replace('<%s>' % found_tag , tags[found_tag])
+					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
 					done.append(found_tag)
 					m = t.search(value, m.start())
@@ -443,7 +467,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo):
+	def replaceTag(cls, query, aInfo, conditional=''):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -459,7 +483,7 @@ class CommandAction(ActionBase):
 			`query` string with tags replaced.
 		"""
 		string = query
-		aInfo = cls.substituteRecursiveTags(aInfo)
+		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
 				value = str(aInfo[tag])			  # assure string
@@ -469,10 +493,10 @@ class CommandAction(ActionBase):
 					value = cls.escapeTag(value)
 				string = string.replace('<' + tag + '>', value)
 		# New line
-		string = string.replace("<br>", '\n')
+		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
 		return string
 
-	def _processCmd(self, cmd, aInfo = None):
+	def _processCmd(self, cmd, aInfo=None, conditional=''):
 		"""Executes a command with preliminary checks and substitutions.
 
 		Before executing any commands, executes the "check" command first
@@ -496,7 +520,17 @@ class CommandAction(ActionBase):
 			self._logSys.debug("Nothing to do")
 			return True
 
-		checkCmd = self.replaceTag(self.actioncheck, self._properties)
+		if conditional == '':
+			conditional = 'family=inet4'
+			if allowed_ipv6:
+				try:
+					ip = aInfo["ip"]
+					if ip and asip(ip).isIPv6:
+						conditional = 'family=inet6'
+				except KeyError:
+					pass
+
+		checkCmd = self.replaceTag('<actioncheck>', self._properties, conditional=conditional)
 		if not self.executeCmd(checkCmd, self.timeout):
 			self._logSys.error(
 				"Invariant check failed. Trying to restore a sane environment")
@@ -506,15 +540,15 @@ class CommandAction(ActionBase):
 				self._logSys.critical("Unable to restore environment")
 				return False
 
+		# Replace static fields
+		realCmd = self.replaceTag(cmd, self._properties, conditional=conditional)
+
 		# Replace tags
-		if not aInfo is None:
-			realCmd = self.replaceTag(cmd, aInfo)
+		if aInfo is not None:
+			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
 		else:
 			realCmd = cmd
 
-		# Replace static fields
-		realCmd = self.replaceTag(realCmd, self._properties)
-
 		return self.executeCmd(realCmd, self.timeout)
 
 	@staticmethod
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 1872eb1f..d0c1495a 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -54,12 +54,17 @@ class CommandActionTest(LogCaptureTestCase):
 			'xyz': "890 <ABC>",
 		}
 		# Recursion is bad
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# Unresolveable substition
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
@@ -127,6 +132,7 @@ class CommandActionTest(LogCaptureTestCase):
 				CallingMap(matches=lambda: str(10))),
 			"09 10 11")
 
+	def testReplaceNoTag(self):
 		# As tag not present, therefore callable should not be called
 		# Will raise ValueError if it is
 		self.assertEqual(
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index b5a9b8d5..44e06067 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1416,7 +1416,6 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip4[0], None)
 		self.assertTrue(ip4[0] is not None)
 		self.assertFalse(ip4[0] is None)
-		self.assertLess(None, ip4[0])
 		self.assertLess(ip4[0], ip4[1])
 		self.assertLess(ip4[1], ip4[2])
 		self.assertEqual(sorted(reversed(ip4)), ip4)
@@ -1424,7 +1423,6 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip6[0], None)
 		self.assertTrue(ip6[0] is not None)
 		self.assertFalse(ip6[0] is None)
-		self.assertLess(None, ip6[0])
 		self.assertLess(ip6[0], ip6[1])
 		self.assertLess(ip6[1], ip6[2])
 		self.assertEqual(sorted(reversed(ip6)), ip6)

From 941a2b6c82a392c94dac7419a1a92e1600b5dbde Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 20:00:24 +0200
Subject: [PATCH 061/623] clean up unnecessarily resp. directly unused action
 properties, because they are ambiguous now; implemented caching functionality
 for same substitutions inside replaceTag: very actual and extreme performance
 growth (up to 1000 times) for ban/unban because too slow
 substituteRecursiveTags by several tags and many includes, but totally
 unnecessary as long as parameters are not changing;

---
 fail2ban/server/action.py        | 182 ++++++++++++++-----------------
 fail2ban/tests/actiontestcase.py |  42 +++++++
 2 files changed, 123 insertions(+), 101 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index cfa03403..cdd4e28c 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -33,6 +33,7 @@ from abc import ABCMeta
 from collections import MutableMapping
 
 from .ipdns import asip
+from .mytime import MyTime
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -203,36 +204,40 @@ class CommandAction(ActionBase):
 
 	_escapedTags = set(('matches', 'ipmatches', 'ipjailmatches'))
 
+	timeout = 60
+	## Command executed in order to initialize the system.
+	actionstart = ''
+	## Command executed when an IP address gets banned.
+	actionban = ''
+	## Command executed when an IP address gets removed.
+	actionunban = ''
+	## Command executed in order to check requirements.
+	actioncheck = ''
+	## Command executed in order to stop the system.
+	actionstop = ''
+
 	def __init__(self, jail, name):
 		super(CommandAction, self).__init__(jail, name)
-		self.timeout = 60
-		## Command executed in order to initialize the system.
-		self.actionstart = ''
-		## Command executed when an IP address gets banned.
-		self.actionban = ''
-		## Command executed when an IP address gets removed.
-		self.actionunban = ''
-		## Command executed in order to check requirements.
-		self.actioncheck = ''
-		## Command executed in order to stop the system.
-		self.actionstop = ''
+		self.__properties = None
+		self.__substCache = {}
 		self._logSys.debug("Created %s" % self.__class__)
 
 	@classmethod
 	def __subclasshook__(cls, C):
 		return NotImplemented # Standard checks
 
-	@property
-	def timeout(self):
-		"""Time out period in seconds for execution of commands.
-		"""
-		return self._timeout
-
-	@timeout.setter
-	def timeout(self, timeout):
-		self._timeout = int(timeout)
-		self._logSys.debug("Set action %s timeout = %i" %
-			(self._name, self.timeout))
+	def __setattr__(self, name, value):
+		if not name.startswith('_') and not callable(value):
+			# special case for some pasrameters:
+			if name == 'timeout':
+				value = MyTime.str2seconds(value)
+			# parameters changed - clear properties and substitution cache:
+			self.__properties = None
+			self.__substCache.clear()
+			#self._logSys.debug("Set action %r %s = %r", self._name, name, value)
+			self._logSys.debug("  Set %s = %r", name, value)
+		# set:
+		self.__dict__[name] = value
 
 	@property
 	def _properties(self):
@@ -240,21 +245,20 @@ class CommandAction(ActionBase):
 
 		This is used to subsitute "tags" in the commands.
 		"""
-		return dict(
+		# if we have a properties - return it:
+		if self.__properties is not None:
+			return self.__properties
+		# otherwise retrieve:
+		self.__properties = dict(
 			(key, getattr(self, key))
 			for key in dir(self)
 			if not key.startswith("_") and not callable(getattr(self, key)))
+		#
+		return self.__properties
 
 	@property
-	def actionstart(self):
-		"""The command executed on start of the jail/action.
-		"""
-		return self._actionstart
-
-	@actionstart.setter
-	def actionstart(self, value):
-		self._actionstart = value
-		self._logSys.debug("Set actionstart = %s" % value)
+	def _substCache(self):
+		return self.__substCache
 
 	def start(self):
 		"""Executes the "actionstart" command.
@@ -263,29 +267,21 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		# check valid tags in properties (raises ValueError if self recursion, etc.):
-		if self._properties:
-			self.substituteRecursiveTags(self._properties)
-		# common (resp. ipv4):
-		startCmd = self.replaceTag('<actionstart>', self._properties, conditional='family=inet4')
-		res = self.executeCmd(startCmd, self.timeout)
-		# start ipv6 actions if available:
-		if allowed_ipv6:
-			startCmd6 = self.replaceTag('<actionstart>', self._properties, conditional='family=inet6')
-			if startCmd6 != startCmd:
-				res &= self.executeCmd(startCmd6, self.timeout)
-		if not res:
-			raise RuntimeError("Error starting action")
-
-	@property
-	def actionban(self):
-		"""The command used when a ban occurs.
-		"""
-		return self._actionban
-
-	@actionban.setter
-	def actionban(self, value):
-		self._actionban = value
-		self._logSys.debug("Set actionban = %s" % value)
+		try:
+			# common (resp. ipv4):
+			startCmd = self.replaceTag('<actionstart>', self._properties, 
+				conditional='family=inet4', cache=self.__substCache)
+			res = self.executeCmd(startCmd, self.timeout)
+			# start ipv6 actions if available:
+			if allowed_ipv6:
+				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
+					conditional='family=inet6', cache=self.__substCache)
+				if startCmd6 != startCmd:
+					res &= self.executeCmd(startCmd6, self.timeout)
+			if not res:
+				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
+		except ValueError, e:
+			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -302,17 +298,6 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
-	@property
-	def actionunban(self):
-		"""The command used when an unban occurs.
-		"""
-		return self._actionunban
-
-	@actionunban.setter
-	def actionunban(self, value):
-		self._actionunban = value
-		self._logSys.debug("Set actionunban = %s" % value)
-
 	def unban(self, aInfo):
 		"""Executes the "actionunban" command.
 
@@ -328,32 +313,6 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
-	@property
-	def actioncheck(self):
-		"""The command used to check the environment.
-
-		This is used prior to a ban taking place to ensure the
-		environment is appropriate. If this check fails, `stop` and
-		`start` is executed prior to the check being called again.
-		"""
-		return self._actioncheck
-
-	@actioncheck.setter
-	def actioncheck(self, value):
-		self._actioncheck = value
-		self._logSys.debug("Set actioncheck = %s" % value)
-
-	@property
-	def actionstop(self):
-		"""The command executed when the jail/actions stops.
-		"""
-		return self._actionstop
-
-	@actionstop.setter
-	def actionstop(self, value):
-		self._actionstop = value
-		self._logSys.debug("Set actionstop = %s" % value)
-
 	def stop(self):
 		"""Executes the "actionstop" command.
 
@@ -361,18 +320,20 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		# common (resp. ipv4):
-		stopCmd = self.replaceTag('<actionstop>', self._properties, conditional='family=inet4')
+		stopCmd = self.replaceTag('<actionstop>', self._properties, 
+			conditional='family=inet4', cache=self.__substCache)
 		res = self.executeCmd(stopCmd, self.timeout)
 		# ipv6 actions if available:
 		if allowed_ipv6:
-			stopCmd6 = self.replaceTag('<actionstop>', self._properties, conditional='family=inet6')
+			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
+				conditional='family=inet6', cache=self.__substCache)
 			if stopCmd6 != stopCmd:
 				res &= self.executeCmd(stopCmd6, self.timeout)
 		if not res:
 			raise RuntimeError("Error stopping action")
 
 	@classmethod
-	def substituteRecursiveTags(cls, tags, conditional=''):
+	def substituteRecursiveTags(cls, inptags, conditional=''):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -382,7 +343,7 @@ class CommandAction(ActionBase):
 
 		Parameters
 		----------
-		tags : dict
+		inptags : dict
 			Dictionary of tags(keys) and their values.
 
 		Returns
@@ -391,6 +352,8 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
+		# copy return tags dict to prevent modifying of inptags:
+		tags = inptags.copy()
 		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
 		while True:
@@ -467,7 +430,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo, conditional=''):
+	def replaceTag(cls, query, aInfo, conditional='', cache=None):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -482,18 +445,32 @@ class CommandAction(ActionBase):
 		str
 			`query` string with tags replaced.
 		"""
+		# use cache if allowed:
+		if cache is not None:
+			ckey = (query, conditional)
+			string = cache.get(ckey)
+			if string is not None:
+				return string
+		# replace:
 		string = query
 		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
-				value = str(aInfo[tag])			  # assure string
+				value = aInfo.get(tag + '?' + conditional)
+				if value is None:
+					value = aInfo.get(tag)
+				value = str(value)			  # assure string
 				if tag in cls._escapedTags:
 					# That one needs to be escaped since its content is
 					# out of our control
 					value = cls.escapeTag(value)
 				string = string.replace('<' + tag + '>', value)
-		# New line
+		# New line, space
 		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
+		# cache if properties:
+		if cache is not None:
+			cache[ckey] = string
+		#
 		return string
 
 	def _processCmd(self, cmd, aInfo=None, conditional=''):
@@ -520,6 +497,7 @@ class CommandAction(ActionBase):
 			self._logSys.debug("Nothing to do")
 			return True
 
+		# conditional corresponding family of the given ip:
 		if conditional == '':
 			conditional = 'family=inet4'
 			if allowed_ipv6:
@@ -530,7 +508,8 @@ class CommandAction(ActionBase):
 				except KeyError:
 					pass
 
-		checkCmd = self.replaceTag('<actioncheck>', self._properties, conditional=conditional)
+		checkCmd = self.replaceTag('<actioncheck>', self._properties, 
+			conditional=conditional, cache=self.__substCache)
 		if not self.executeCmd(checkCmd, self.timeout):
 			self._logSys.error(
 				"Invariant check failed. Trying to restore a sane environment")
@@ -541,7 +520,8 @@ class CommandAction(ActionBase):
 				return False
 
 		# Replace static fields
-		realCmd = self.replaceTag(cmd, self._properties, conditional=conditional)
+		realCmd = self.replaceTag(cmd, self._properties, 
+			conditional=conditional, cache=self.__substCache)
 
 		# Replace tags
 		if aInfo is not None:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index d0c1495a..4a34a305 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -139,6 +139,48 @@ class CommandActionTest(LogCaptureTestCase):
 			self.__action.replaceTag("abc",
 				CallingMap(matches=lambda: int("a"))), "abc")
 
+	def testReplaceTagConditionalCached(self):
+		setattr(self.__action, 'abc', "123")
+		setattr(self.__action, 'abc?family=inet4', "345")
+		setattr(self.__action, 'abc?family=inet6', "567")
+		setattr(self.__action, 'xyz', "890-<abc>")
+		setattr(self.__action, 'banaction', "Text <xyz> text <abc>")
+		# test replacement in sub tags and direct, conditional, cached:
+		cache = self.__action._substCache
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 890-123 text 123")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 890-345 text 345")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 890-567 text 567")
+		self.assertEqual(len(cache) if cache is not None else -1, 3)
+		# set one parameter - internal properties and cache should be reseted:
+		setattr(self.__action, 'xyz', "000-<abc>")
+		self.assertEqual(len(cache) if cache is not None else -1, 0)
+		# test againg, should have 000 instead of 890:
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 000-123 text 123")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 000-345 text 345")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction>", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 000-567 text 567")
+		self.assertEqual(len(cache), 3)
+
+
 	def testExecuteActionBan(self):
 		self.__action.actionstart = "touch /tmp/fail2ban.test"
 		self.assertEqual(self.__action.actionstart, "touch /tmp/fail2ban.test")

From 1b21f21c22fceedee7c178236c5d432c46c2bddd Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Fri, 6 May 2016 16:04:21 +0200
Subject: [PATCH 062/623] CIDR represenation if IPAddr object is a network
 prefix

---
 fail2ban/server/ipdns.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index da820366..cd22172a 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -203,10 +203,15 @@ class IPAddr(object):
 			self._raw = ipstring
 
 	def __repr__(self):
-		return self.ntoa
+		if self.isIPv4 and self.plen < 32:
+			return "%s/%d" % (self.ntoa, self.plen)
+		elif self.isIPv6 and self.plen < 128:
+			return "%s/%d" % (self.ntoa, self.plen)
+		else:
+			return self.ntoa
 
 	def __str__(self):
-		return self.ntoa
+		return self.__repr__()
 	
 	@property
 	def addr(self):

From d65e37e93d036dce73d18235f47ace66b5486c77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 21:46:04 +0200
Subject: [PATCH 063/623] CIDR splitting functionality moved from filter to
 IPAddr; meantime commit: code review, simplification, pythonization, etc. +
 test cases extended

---
 fail2ban/server/filter.py        | 12 ++-----
 fail2ban/server/ipdns.py         | 54 +++++++++++++++++++++-----------
 fail2ban/tests/filtertestcase.py | 11 +++++++
 3 files changed, 48 insertions(+), 29 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 4bbf7996..59d2e0d7 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -343,19 +343,11 @@ class Filter(JailThread):
 		# An empty string is always false
 		if ipstr == "":
 			return
-		s = ipstr.split('/', 1)
-		# IP address without CIDR mask
-		if len(s) == 1:
-			s.insert(1, -1) # <0 means no CIDR
-		elif "." in s[1]: # 255.255.255.0 style mask
-			s[1] = IPAddr.masktoplen(s[1])
-		s[1] = long(s[1])
-		
 		# Create IP address object
-		ip = IPAddr(s[0], s[1])
+		ip = IPAddr(ipstr)
 
 		# log and append to ignore list
-		logSys.debug("Add %r to ignore list (%r, %r)", ip, s[0], s[1])
+		logSys.debug("Add %r to ignore list (%r)", ip, ipstr)
 		self.__ignoreIpList.append(ip)
 
 	def delIgnoreIP(self, ip):
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index cd22172a..8021b1d8 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -150,8 +150,20 @@ class IPAddr(object):
 		IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
-	# object methods
-	def __init(self, ipstring, cidr=-1):
+	@staticmethod
+	def __wrap_ipstr(ipstr):
+		if "/" not in ipstr:
+			return ipstr, -1
+		s = ipstr.split('/', 1)
+		# IP address without CIDR mask
+		if len(s) > 2:
+			raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,))
+		if "." in s[1]: # 255.255.255.0 style mask
+			s[1] = IPAddr.masktoplen(s[1])
+		s[1] = long(s[1])
+		return s
+		
+	def __init(self, ipstr, cidr=-1):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -160,9 +172,13 @@ class IPAddr(object):
 		self._plen = 0
 		self._maskplen = None
 		self._raw = ""
+
+		if cidr == -1:
+			ipstr, cidr = self.__wrap_ipstr(ipstr)
+
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
-				binary = socket.inet_pton(family, ipstring)
+				binary = socket.inet_pton(family, ipstr)
 				self._family = family
 				break
 			except socket.error:
@@ -200,18 +216,13 @@ class IPAddr(object):
 			# string couldn't be converted neither to a IPv4 nor
 			# to a IPv6 address - retain raw input for later use
 			# (e.g. DNS resolution)
-			self._raw = ipstring
+			self._raw = ipstr
 
 	def __repr__(self):
-		if self.isIPv4 and self.plen < 32:
-			return "%s/%d" % (self.ntoa, self.plen)
-		elif self.isIPv6 and self.plen < 128:
-			return "%s/%d" % (self.ntoa, self.plen)
-		else:
-			return self.ntoa
+		return self.ntoa
 
 	def __str__(self):
-		return self.__repr__()
+		return self.ntoa
 	
 	@property
 	def addr(self):
@@ -280,10 +291,10 @@ class IPAddr(object):
 	def hexdump(self):
 		"""Hex representation of the IP address (for debug purposes)
 		"""
-		if self.family == socket.AF_INET:
-			return "%08x" % self.addr
-		elif self.family == socket.AF_INET6:
-			return "%032x" % self.addr
+		if self._family == socket.AF_INET:
+			return "%08x" % self._addr
+		elif self._family == socket.AF_INET6:
+			return "%032x" % self._addr
 		else:
 			return ""
 
@@ -293,18 +304,23 @@ class IPAddr(object):
 		""" represent IP object as text like the deprecated
 			C pendant inet.ntoa but address family independent
 		"""
+		add = ''
 		if self.isIPv4:
 			# convert network to host byte order
 			binary = struct.pack("!L", self._addr)
+			if self._plen and self._plen < 32:
+				add = "/%d" % self._plen
 		elif self.isIPv6:
 			# convert network to host byte order
-			hi = self.addr >> 64
-			lo = self.addr & 0xFFFFFFFFFFFFFFFFL
+			hi = self._addr >> 64
+			lo = self._addr & 0xFFFFFFFFFFFFFFFFL
 			binary = struct.pack("!QQ", hi, lo)
+			if self._plen and self._plen < 128:
+				add = "/%d" % self._plen
 		else:
 			return self._raw
-
-		return socket.inet_ntop(self.family, binary)
+		
+		return socket.inet_ntop(self._family, binary) + add
 
 	def getPTR(self, suffix=""):
 		""" return the DNS PTR string of the provided IP address object
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 44e06067..2d82ab6f 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1446,6 +1446,17 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		# compare with string direct:
 		self.assertEqual(d, d2)
 
+	def testIPAddr_CIDR(self):
+		self.assertEqual(str(IPAddr('93.184.0.1', 24)), '93.184.0.0/24')
+		self.assertEqual(str(IPAddr('192.168.1.0/255.255.255.128')), '192.168.1.0/25')
+		self.assertEqual(IPAddr('93.184.0.1', 24).ntoa, '93.184.0.0/24')
+		self.assertEqual(IPAddr('192.168.1.0/255.255.255.128').ntoa, '192.168.1.0/25')
+
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8::', 120)), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8::', 120).ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+
 	def testIPAddr_CompareDNS(self):
 		ips = IPAddr('example.com')
 		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))

From 50e5a7e53811ff60cb2546597ecb7748b1675a6a Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 8 May 2016 09:09:12 +0200
Subject: [PATCH 064/623] More beautifier tests

---
 fail2ban/tests/clientbeautifiertestcase.py | 178 +++++++++++++++++++--
 1 file changed, 164 insertions(+), 14 deletions(-)

diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 397bce1d..6fc63e6f 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -26,6 +26,7 @@ import unittest
 from ..client.beautifier import Beautifier
 from ..version import version
 from ..server.ipdns import IPAddr
+from ..exceptions import UnknownJailException, DuplicateJailException
 
 class BeautifierTest(unittest.TestCase):
 
@@ -57,10 +58,6 @@ class BeautifierTest(unittest.TestCase):
 		self.b.setInputCmd(["start"])
 		self.assertEqual(self.b.beautify(None), "Jail started")
 
-	def testFlushLogs(self):
-		self.b.setInputCmd(["flushlogs"])
-		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
-
 	def testStopJail(self):
 		self.b.setInputCmd(["stop", "ssh"])
 		self.assertEqual(self.b.beautify(None), "Jail stopped")
@@ -95,15 +92,168 @@ class BeautifierTest(unittest.TestCase):
 						]
 					)
 				)
-		output = """Status for the jail: ssh
-|- Filter
-|  |- Currently failed:	0
-|  |- Total failed:	0
-|  `- File list:	/var/log/auth.log
-`- Actions
-   |- Currently banned:	3
-   |- Total banned:	3
-   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"""
-
+		output = "Status for the jail: ssh\n"
+		output += "|- Filter\n"
+		output += "|  |- Currently failed:	0\n"
+		output += "|  |- Total failed:	0\n"
+		output += "|  `- File list:	/var/log/auth.log\n"
+		output += "`- Actions\n"
+		output += "   |- Currently banned:	3\n"
+		output += "   |- Total banned:	3\n"
+		output += "   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"
 		self.assertEqual(self.b.beautify(response), output)
 
+	def testFlushLogs(self):
+		self.b.setInputCmd(["flushlogs"])
+		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
+	
+	def testSyslogSocket(self):
+		self.b.setInputCmd(["get", "syslogsocket"])
+		output = "Current syslog socket is:\n`- auto"
+		self.assertEqual(self.b.beautify("auto"), output)
+
+	def testLogTarget(self):
+		self.b.setInputCmd(["get", "logtarget"])
+		output = "Current logging target is:\n`- /var/log/fail2ban.log"
+		self.assertEqual(self.b.beautify("/var/log/fail2ban.log"), output)
+
+	def testLogLevel(self):
+		self.b.setInputCmd(["get", "loglevel"])
+		output = "Current logging level is 'INFO'"
+		self.assertEqual(self.b.beautify("INFO"), output)
+
+	def testDbFile(self):
+		self.b.setInputCmd(["get", "dbfile"])
+		response = "/var/lib/fail2ban/fail2ban.sqlite3"
+		output = "Current database file is:\n`- " + response
+		self.assertEqual(self.b.beautify(response), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testDbPurgeAge(self):
+		self.b.setInputCmd(["get", "dbpurgeage"])
+		output = "Current database purge age is:\n`- 86400seconds"
+		self.assertEqual(self.b.beautify(86400), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testLogPath(self):
+		self.b.setInputCmd(["get", "sshd", "logpath"])
+		response = []
+		output = "No file is currently monitored"
+		self.assertEqual(self.b.beautify(response), output)
+		response = ["/var/log/auth.log"]
+		output = "Current monitored log file(s):\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "addlogpath", "/var/log/messages"])
+		response = ["/var/log/messages", "/var/log/auth.log"]
+		outputadd = "Current monitored log file(s):\n"
+		outputadd += "|- /var/log/messages\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), outputadd)
+
+		self.b.setInputCmd(["set", "sshd", "dellogpath", "/var/log/messages"])
+		response = ["/var/log/auth.log"]
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testLogEncoding(self):
+		self.b.setInputCmd(["get", "sshd", "logencoding"])
+		output = "Current log encoding is set to:\nUTF-8"
+		self.assertEqual(self.b.beautify("UTF-8"), output)
+
+	def testJournalMatch(self):
+		self.b.setInputCmd(["get", "sshd", "journalmatch"])
+		self.assertEqual(self.b.beautify([]), "No journal match filter set")
+
+		self.b.setInputCmd(["set", "sshd", "addjournalmatch"])
+		response = [["_SYSTEMD_UNIT", "sshd.service"]]
+		output = "Current match filter:\n"
+		output += "_SYSTEMD_UNIT sshd.service"
+		self.assertEqual(self.b.beautify(response), output)
+		
+		response.append(["_COMM", "sshd"])
+		output += " + _COMM sshd"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "deljournalmatch"])
+		response.remove(response[1])
+		self.assertEqual(self.b.beautify(response), output.split(" + ")[0])
+
+	def testDatePattern(self):
+		self.b.setInputCmd(["get", "sshd", "datepattern"])
+		output = "Current date pattern set to: "
+		response = (None, "Default Detectors")
+		self.assertEqual(self.b.beautify(None), 
+				output + "Not set/required")
+		self.assertEqual(self.b.beautify(response), 
+				output + "Default Detectors")
+		self.assertEqual(self.b.beautify(("test", "test")), 
+				output + "test (test)")
+
+	def testIgnoreIP(self):
+		self.b.setInputCmd(["get", "sshd", "ignoreip"])
+		output = "No IP address/network is ignored"
+		self.assertEqual(self.b.beautify([]), output)
+
+		self.b.setInputCmd(["set", "sshd", "addignoreip"])
+		response = [
+			IPAddr("127.0.0.0", 8), 
+			IPAddr("::1"), 
+			IPAddr("2001:db8::", 32), 
+			IPAddr("::ffff:10.0.2.1")
+		]
+		output = "These IP addresses/networks are ignored:\n"
+		output += "|- 127.0.0.0/8\n|- ::1\n"
+		output += "|- 2001:db8::/32\n`- 10.0.2.1"
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testFailRegex(self):
+		self.b.setInputCmd(["get", "sshd", "failregex"])
+		output = "No regular expression is defined"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The following regular expression are defined:\n"
+		output += "|- [0]: ^$\n`- [1]: .*"
+		self.assertEqual(self.b.beautify(["^$", ".*"]), output)
+
+	def testActions(self):
+		self.b.setInputCmd(["get", "sshd", "actions"])
+		output = "No actions for jail sshd"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The jail sshd has the following actions:\n"
+		output += "iptables-multiport"
+		self.assertEqual(self.b.beautify(["iptables-multiport"]), output)
+
+	def testActionProperties(self):
+		self.b.setInputCmd(["get", "sshd", "actionproperties", "iptables"])
+		output = "No properties for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following properties:"
+		output += "\nactionban, actionunban"
+		response = ("actionban", "actionunban")
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testActionMethods(self):
+		self.b.setInputCmd(["get", "sshd", "actionmethods", "iptables"])
+		output = "No methods for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following methods:\n"
+		output += "ban, unban"
+		self.assertEqual(self.b.beautify(["ban", "unban"]), output)
+	
+#	def testException(self):
+#		self.b.setInputCmd(["get", "sshd", "logpath"])
+#		self.assertRaises(self.b.beautify(1), TypeError)
+
+	def testBeautifyError(self):
+		response = UnknownJailException("sshd")
+		output = "Sorry but the jail 'sshd' does not exist"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		response = DuplicateJailException("sshd")
+		output = "The jail 'sshd' already exists"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		output = "Sorry but the command is invalid"
+		self.assertEqual(self.b.beautifyError(IndexError()), output)

From 9da7bb1068d6a27497a72fc8e1691881866777d6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 22:09:14 +0200
Subject: [PATCH 065/623] meantime commit: code review, simplification,
 pythonization, etc. (test cases passed)

---
 fail2ban/client/beautifier.py              | 42 +++++++++++-----------
 fail2ban/tests/clientbeautifiertestcase.py |  6 ++--
 fail2ban/tests/filtertestcase.py           |  5 +++
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 308cb035..7b27e92f 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -89,45 +89,45 @@ class Beautifier:
 				msg = "\n".join(msg)
 			elif inC[1] == "syslogsocket":
 				msg = "Current syslog socket is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1] == "logtarget":
 				msg = "Current logging target is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1:2] == ['loglevel']:
 				msg = "Current logging level is "
 				if response == 1:
-					msg = msg + "ERROR"
+					msg += "ERROR"
 				elif response == 2:
-					msg = msg + "WARN"
+					msg += "WARN"
 				elif response == 3:
-					msg = msg + "INFO"
+					msg += "INFO"
 				elif response == 4:
-					msg = msg + "DEBUG"
+					msg += "DEBUG"
 				else:
-					msg = msg + repr(response)
+					msg += repr(response)
 			elif inC[1] == "dbfile":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database file is:\n"
-					msg = msg + "`- " + response
+					msg += "`- " + response
 			elif inC[1] == "dbpurgeage":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database purge age is:\n"
-					msg = msg + "`- %iseconds" % response
+					msg += "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
 				else:
 					msg = "Current monitored log file(s):\n"
 					for path in response[:-1]:
-						msg = msg + "|- " + path + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + path + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] == "logencoding":
 				msg = "Current log encoding is set to:\n"
-				msg = msg + response
+				msg += response
 			elif inC[2] in ("journalmatch", "addjournalmatch", "deljournalmatch"):
 				if len(response) == 0:
 					msg = "No journal match filter set"
@@ -137,19 +137,19 @@ class Beautifier:
 			elif inC[2] == "datepattern":
 				msg = "Current date pattern set to: "
 				if response is None:
-					msg = msg + "Not set/required"
+					msg += "Not set/required"
 				elif response[0] is None:
-					msg = msg + "%s" % response[1]
+					msg += "%s" % response[1]
 				else:
-					msg = msg + "%s (%s)" % response
+					msg += "%s (%s)" % response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"
 				else:
 					msg = "These IP addresses/networks are ignored:\n"
 					for ip in response[:-1]:
-						msg = msg + "|- " + ip + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + ip + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] in ("failregex", "addfailregex", "delfailregex",
 							"ignoreregex", "addignoreregex", "delignoreregex"):
 				if len(response) == 0:
@@ -157,10 +157,10 @@ class Beautifier:
 				else:
 					msg = "The following regular expression are defined:\n"
 					c = 0
-					for ip in response[:-1]:
-						msg = msg + "|- [" + str(c) + "]: " + ip + "\n"
+					for l in response[:-1]:
+						msg += "|- [" + str(c) + "]: " + l + "\n"
 						c += 1
-					msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1]
+					msg += "`- [" + str(c) + "]: " + response[-1]
 			elif inC[2] == "actions":
 				if len(response) == 0:
 					msg = "No actions for jail %s" % inC[1]
@@ -187,7 +187,7 @@ class Beautifier:
 			logSys.warning("Beautifier error. Please report the error")
 			logSys.error("Beautify " + repr(response) + " with "
 				+ repr(self.__inputCmd) + " failed")
-			msg = msg + repr(response)
+			msg += repr(response)
 		return msg
 
 	def beautifyError(self, response):
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 6fc63e6f..0390a8ff 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -201,8 +201,10 @@ class BeautifierTest(unittest.TestCase):
 			IPAddr("::ffff:10.0.2.1")
 		]
 		output = "These IP addresses/networks are ignored:\n"
-		output += "|- 127.0.0.0/8\n|- ::1\n"
-		output += "|- 2001:db8::/32\n`- 10.0.2.1"
+		output += "|- 127.0.0.0/8\n"
+		output += "|- ::1\n"
+		output += "|- 2001:db8::/32\n"
+		output += "`- 10.0.2.1"
 		self.assertEqual(self.b.beautify(response), output)
 
 	def testFailRegex(self):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 2d82ab6f..e8bd2e37 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1457,6 +1457,11 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 
+	def testIPAddr_CIDR_Repr(self):
+		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
+			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]
+		)
+
 	def testIPAddr_CompareDNS(self):
 		ips = IPAddr('example.com')
 		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))

From e8203dabbac5f937fc5bfd9b3f5e33cd256eeaf8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 May 2016 22:51:40 +0200
Subject: [PATCH 066/623] python 2.6 compatible test cases

---
 fail2ban/tests/filtertestcase.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index e8bd2e37..5e493f83 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1416,20 +1416,20 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertNotEqual(ip4[0], None)
 		self.assertTrue(ip4[0] is not None)
 		self.assertFalse(ip4[0] is None)
-		self.assertLess(ip4[0], ip4[1])
-		self.assertLess(ip4[1], ip4[2])
+		self.assertTrue(ip4[0] < ip4[1])
+		self.assertTrue(ip4[1] < ip4[2])
 		self.assertEqual(sorted(reversed(ip4)), ip4)
 		# ip6
 		self.assertNotEqual(ip6[0], None)
 		self.assertTrue(ip6[0] is not None)
 		self.assertFalse(ip6[0] is None)
-		self.assertLess(ip6[0], ip6[1])
-		self.assertLess(ip6[1], ip6[2])
+		self.assertTrue(ip6[0] < ip6[1])
+		self.assertTrue(ip6[1] < ip6[2])
 		self.assertEqual(sorted(reversed(ip6)), ip6)
 		# ip4 vs ip6
 		self.assertNotEqual(ip4[0], ip6[0])
-		self.assertLess(ip4[0], ip6[0])
-		self.assertLess(ip4[2], ip6[2])
+		self.assertTrue(ip4[0] < ip6[0])
+		self.assertTrue(ip4[2] < ip6[2])
 		self.assertEqual(sorted(reversed(ip4+ip6)), ip4+ip6)
 		# hashing (with string as key):
 		d={

From f47fdf89180de4dcc81ad6076fca2562a9ff2789 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 13:06:16 +0200
Subject: [PATCH 067/623] code coverage: extend conditional test with direct
 replacement inside replaceTag (besides recursive sub tags)

---
 fail2ban/tests/actiontestcase.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 4a34a305..f3778022 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -149,17 +149,17 @@ class CommandActionTest(LogCaptureTestCase):
 		cache = self.__action._substCache
 		for i in range(2):
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="", cache=cache),
-				"Text 890-123 text 123")
+				"Text 890-123 text 123 '123'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet4", cache=cache),
-				"Text 890-345 text 345")
+				"Text 890-345 text 345 '345'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
-				"Text 890-567 text 567")
+				"Text 890-567 text 567 '567'")
 		self.assertEqual(len(cache) if cache is not None else -1, 3)
 		# set one parameter - internal properties and cache should be reseted:
 		setattr(self.__action, 'xyz', "000-<abc>")
@@ -167,17 +167,17 @@ class CommandActionTest(LogCaptureTestCase):
 		# test againg, should have 000 instead of 890:
 		for i in range(2):
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="", cache=cache),
-				"Text 000-123 text 123")
+				"Text 000-123 text 123 '123'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet4", cache=cache),
-				"Text 000-345 text 345")
+				"Text 000-345 text 345 '345'")
 			self.assertEqual(
-				self.__action.replaceTag("<banaction>", self.__action._properties, 
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
-				"Text 000-567 text 567")
+				"Text 000-567 text 567 '567'")
 		self.assertEqual(len(cache), 3)
 
 

From 23ad50dbaab5c9826bbda3115a3ecb858f4fe459 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 15:27:25 +0200
Subject: [PATCH 068/623] correct <HOST> expression in failregex (now IPv6 will
 be really found from filter); respect standard spelling of IPv6 (with port)
 enclosed in brackets ([ipv6]:port), to prevent the necessarily changing of
 many filter definitions, comparison: #IPv4: 127.0.0.1:55555 #IPv6:
 [a:b:c:d::1]:55555 some small fixes (in test cases also) and code review and
 few optimizations;

---
 fail2ban/server/failregex.py          | 18 ++++++++-
 fail2ban/server/ipdns.py              | 53 +++++++++++++++++----------
 fail2ban/tests/files/logs/apache-auth |  4 ++
 fail2ban/tests/files/logs/sshd        |  2 +
 fail2ban/tests/servertestcase.py      |  8 ++--
 fail2ban/tests/utils.py               | 10 +++++
 6 files changed, 69 insertions(+), 26 deletions(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 75e64c46..6076acc3 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -43,8 +43,8 @@ class Regex:
 	def __init__(self, regex):
 		self._matchCache = None
 		# Perform shortcuts expansions.
-		# Replace "<HOST>" with default regular expression for host.
-		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]*\w)")
+		# Resolve "<HOST>" tag using default regular expression for host:
+		regex = Regex._resolveHostTag(regex)
 		# Replace "<SKIPLINES>" with regular expression for multiple lines.
 		regexSplit = regex.split("<SKIPLINES>")
 		regex = regexSplit[0]
@@ -61,6 +61,20 @@ class Regex:
 
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
+
+	@staticmethod
+	def _resolveHostTag(regex):
+		# Replace "<HOST>" with default regular expression for host:
+		# Other candidates (see gh-1374 for the discussion about):
+		#   differentiate: r"""(?:(?:::f{4,6}:)?(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<HOST>[\w\-.^_]*\w))"""
+		#     expected many changes in filter, failregex, etc...
+		#   simple:        r"""(?:::f{4,6}:)?(?P<host>[\w\-.^_:]*\w)"""
+		#     not good enough, if not precise expressions around <HOST>, because for example will match '1.2.3.4:23930' as ip-address;
+		# Todo: move this functionality to filter reader, as default <HOST> replacement,
+		#       make it configurable (via jail/filter configs)
+		return regex.replace("<HOST>",
+		 	r"""(?:::f{4,6}:)?(?P<host>(?:\d{1,3}\.){3}\d{1,3}|\[?(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}\]?|(?<=:):)|[\w\-.^_]*\w)""")
+
 	##
 	# Gets the regular expression.
 	#
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 8021b1d8..bca03428 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -103,7 +103,7 @@ class DNSUtils:
 		# Search for plain IP
 		plainIP = IPAddr.searchIP(text)
 		if plainIP is not None:
-			ip = IPAddr(plainIP.group(0))
+			ip = IPAddr(plainIP)
 			if ip.isValid:
 				ipList.append(ip)
 
@@ -127,9 +127,8 @@ class DNSUtils:
 class IPAddr(object):
 	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
-
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-	IP6_CRE = re.compile("^[0-9a-fA-F]{4}[0-9a-fA-F:]+:[0-9a-fA-F]{1,4}|::1$")
+	IP_4_6_CRE = re.compile(
+	  r"""^(?:(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?)$""")
 	# An IPv4 compatible IPv6 to be reused (see below)
 	IP6_4COMPAT = None
 
@@ -139,19 +138,33 @@ class IPAddr(object):
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
 
-	def __new__(cls, ipstring, cidr=-1):
-		# already correct IPAddr
-		args = (ipstring, cidr)
+	def __new__(cls, ipstr, cidr=-1):
+		# check already cached as IPAddr
+		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
 		if ip is not None:
 			return ip
+		# wrap mask to cidr (correct plen):
+		if cidr == -1:
+			ipstr, cidr = IPAddr.__wrap_ipstr(ipstr)
+			args = (ipstr, cidr)
+			# check cache again:
+			if cidr != -1:
+				ip = IPAddr.CACHE_OBJ.get(args)
+				if ip is not None:
+					return ip
 		ip = super(IPAddr, cls).__new__(cls)
-		ip.__init(ipstring, cidr)
+		ip.__init(ipstr, cidr)
 		IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
 	@staticmethod
 	def __wrap_ipstr(ipstr):
+		# because of standard spelling of IPv6 (with port) enclosed in brackets ([ipv6]:port),
+		# remove they now (be sure the <HOST> inside failregex uses this for IPv6 (has \[?...\]?)
+		if len(ipstr) > 2 and ipstr[0] == '[' and ipstr[-1] == ']':
+			ipstr = ipstr[1:-1]
+		# test mask:
 		if "/" not in ipstr:
 			return ipstr, -1
 		s = ipstr.split('/', 1)
@@ -173,9 +186,6 @@ class IPAddr(object):
 		self._maskplen = None
 		self._raw = ""
 
-		if cidr == -1:
-			ipstr, cidr = self.__wrap_ipstr(ipstr)
-
 		for family in [socket.AF_INET, socket.AF_INET6]:
 			try:
 				binary = socket.inet_pton(family, ipstr)
@@ -376,17 +386,17 @@ class IPAddr(object):
 
 	@property
 	def maskplen(self):
-		plen = 0
+		mplen = 0
 		if self._maskplen is not None:
-			return self._plen
-		maddr = self.addr
+			return self._maskplen
+		maddr = self._addr
 		while maddr:
 			if not (maddr & 0x80000000):
 				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
 			maddr = (maddr << 1) & 0xFFFFFFFFL
-			plen += 1
-		self._maskplen = plen
-		return plen
+			mplen += 1
+		self._maskplen = mplen
+		return mplen
 		
 	@staticmethod
 	def masktoplen(mask):
@@ -400,10 +410,13 @@ class IPAddr(object):
 	def searchIP(text):
 		"""Search if text is an IP address, and return it if so, else None
 		"""
-		match = IPAddr.IP_CRE.match(text)
+		match = IPAddr.IP_4_6_CRE.match(text)
 		if not match:
-			match = IPAddr.IP6_CRE.match(text)
-		return match if match else None
+			return None
+		ipstr = match.group('IPv4')
+		if ipstr != '':
+			return ipstr
+		return match.group('IPv6')
 
 
 # An IPv4 compatible IPv6 to be reused
diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth
index 5b7b3c48..29de57eb 100644
--- a/fail2ban/tests/files/logs/apache-auth
+++ b/fail2ban/tests/files/logs/apache-auth
@@ -8,6 +8,8 @@
 
 # failJSON: { "time": "2013-07-11T01:21:43", "match": true , "host": "194.228.20.113" } 
 [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: /
+# failJSON: { "time": "2013-07-11T01:21:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Thu Jul 11 01:21:44 2013] [error] [client 2606:2800:220:1:248:1893:25c8:1946] user test-ipv6 not found: /
 
 # The failures below use the configuration described in fail2ban/tests/files/config/apache-auth
 #
@@ -56,6 +58,8 @@
 
 # failJSON: { "time": "2013-07-20T22:11:43", "match": true , "host": "127.0.0.1" } 
 [Sat Jul 20 22:11:43.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client 127.0.0.1:51548] AH01637: Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
+# failJSON: { "time": "2013-07-20T22:11:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Sat Jul 20 22:11:44.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client [2606:2800:220:1:248:1893:25c8:1946]:51548] AH01637: Authorization of user test-ipv6 to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
 
 # wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null
 # failJSON: { "time": "2013-07-20T21:42:44", "match": true , "host": "127.0.0.1" } 
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index 7baf4be7..89e4a3e2 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -3,6 +3,8 @@
 Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6
 # failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" }
 May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com
+# failJSON: { "time": "2005-05-29T20:56:56", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" }
+May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-ipv6 from 2606:2800:220:1:248:1893:25c8:1946
 
 #2
 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" }
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 893976be..1c05f5e8 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -458,9 +458,9 @@ class Transmitter(TransmitterBase):
 				"failed attempt from <HOST> again",
 			],
 			[
-				"user john at (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"failed attempt from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w) again",
+				"user john at %s" % (Regex._resolveHostTag('<HOST>')),
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
+				"failed attempt from %s again" % (Regex._resolveHostTag('<HOST>')),
 			],
 			self.jailName
 		)
@@ -483,7 +483,7 @@ class Transmitter(TransmitterBase):
 			],
 			[
 				"user john",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
 				"Dont match me!",
 			],
 			self.jailName
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index e97daebf..a0036979 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -101,6 +101,16 @@ def initTests(opts):
 		c.set('192.0.2.%s' % i, None)
 		c.set('198.51.100.%s' % i, None)
 		c.set('203.0.113.%s' % i, None)
+	if unittest.F2B.no_network: # pragma: no cover
+		# precache all wrong dns to ip's used in test cases:
+		c = DNSUtils.CACHE_nameToIp
+		for i in (
+			('999.999.999.999', []),
+			('abcdef.abcdef', []),
+			('192.168.0.', []),
+			('failed.dns.ch', []),
+		):
+			c.set(*i)
 
 
 def mtimesleep():

From c22ba5413dfd82de5ced285304a6eadd595fcdfe Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 16:58:13 +0200
Subject: [PATCH 069/623] changes from ipv6-support-0.10 after merge

---
 ChangeLog | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9c305340..4e272228 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,10 +15,24 @@ TODO: list all the fixes/enhancements/etc which came with the PR #1346
 
 
 - New Features:
-
+   * IPv6 support:
+     - IP addresses are now handled as objects rather than strings capable for 
+       handling both address types IPv4 and IPv6
+     - iptables related actions have been amended to support IPv6 specific actions
+       additionally
+     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
+     - pf action for *BSD systems has been improved and supports now also v4 and v6
+     - name resolution is now working for either address type
+		 - new conditional section functionality used in config resp. includes:
+       - [Init?family=inet4] - IPv4 qualified hosts only
+       - [Init?family=inet6] - IPv6 qualified hosts only
 
 - Enhancements:
-
+   * several functionality of class `DNSUtils` moved to new class `IPAddr`, 
+     both classes moved to new module `ipdns`
+   * pseudo-conditional section introduced, for conditional substitution resp. 
+     evaluation of parameters for different family qualified hosts, 
+     syntax `[Section?family=inet6]` (currently use for IPv6-support only).
 
 ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
 -----------
@@ -92,14 +106,6 @@ ver. 0.9.4 (2016/03/08) - for-you-ladies
    * sshd filter got new failregex to match "maximum authentication
      attempts exceeded" (introduced in openssh 6.8)
    * Added filter for Mac OS screen sharing (VNC) daemon
-   * IPv6 support:
-     - IP addresses are now handled as objects rather than strings capable for 
-       handling both address types IPv4 and IPv6
-     - iptables related actions have been amended to support IPv6 specific actions
-       additionally
-     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
-     - pf action for *BSD systems has been improved and supports now also v4 and v6
-     - Name resolution is now working for either address type
 
 - Enhancements:
    * Do not rotate empty log files

From 3df97beaa6ef8bbf8865c40266c69ed12a45e343 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 18:03:22 +0200
Subject: [PATCH 070/623] changelog entries from sebres:f2b-perfom-prepare-716
 (gh-1346)

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4e272228..e30ea8c6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -9,10 +9,17 @@ Fail2Ban: Changelog
 ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
 -----------
 
-TODO: list all the fixes/enhancements/etc which came with the PR #1346
+TODO: implementing of options resp. other tasks from PR #1346
 
 - Fixes:
-
+   * [grave] memory leak's fixed (gh-1277, gh-1234)
+   * tricky bug fix: last position of log file will be never retrieved (gh-795),
+     because of CASCADE all log entries will be deleted from logs table together with jail, 
+     if used "INSERT OR REPLACE" statement
+   * asyncserver (asyncore) code fixed and test cases repaired (again gh-161)
+   * testSocket: sporadical bug repaired - wait for server thread starts a socket (listener)
+   * testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash,
+     kill tree in any case (gh-1155)
 
 - New Features:
    * IPv6 support:
@@ -28,6 +35,28 @@ TODO: list all the fixes/enhancements/etc which came with the PR #1346
        - [Init?family=inet6] - IPv6 qualified hosts only
 
 - Enhancements:
+   * huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)
+   * datedetector: in-place reordering using hits and last used time: 
+     matchTime, template list etc. rewritten because of performance degradation
+   * prevent out of memory situation if many IP's makes extremely many failures (maxEntries)
+   * introduced string to seconds (str2seconds) for configuration entries with time,
+     use `1h` instead of `3600`, `1d` instead of `86400`, etc
+   * seekToTime - prevent completely read of big files first time (after start of service), 
+     initial seek to start time using half-interval search algorithm (see issue gh-795)
+   * ticket and some other modules prepared to easy merge with newest version of 'ban-time-incr'
+   * cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name,
+     especially for wrong dns or lazy dns-system
+   * FailManager memory-optimization: increases performance, 
+     prevents memory leakage, because don't copy failures list on some operations
+   * fail2ban-testcases - new options introduced:
+     - `-f`, `--fast` to decrease wait intervals, avoid passive waiting, and skip
+       few very slow test cases (implied memory database, see `-m` and no gamin tests `-g`)
+     - `-g`, `--no-gamin` to prevent running of tests that require the gamin (slow)
+     - `-m`, `--memory-db` - run database tests using memory instead of file
+     - `-i`, `--ignore` - negate [regexps] filter to ignore tests matched specified regexps
+   * background servicing: prevents memory leak on some platforms/python versions, using forced GC
+     in periodic intervals (latency and threshold)
+   * executeCmd partially moved from action to new module utils
    * several functionality of class `DNSUtils` moved to new class `IPAddr`, 
      both classes moved to new module `ipdns`
    * pseudo-conditional section introduced, for conditional substitution resp. 

From 9b72522a01a323fb9ae35ed0b58690317682c0aa Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 11 May 2016 18:47:12 +0200
Subject: [PATCH 071/623] todo fix: not necessary in 0.10 branch

---
 fail2ban/tests/servertestcase.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 1c05f5e8..90f7aaf2 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1022,9 +1022,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
 					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
 						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
-					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
-					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
-					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and (
+					# if fast add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
 					):
 						cmd[0] = "set"

From 4ce240ed4055b37b870027a41ba8e432a69dc531 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 9 Feb 2016 14:23:40 +0100
Subject: [PATCH 072/623] try to start server in foreground

# Conflicts:
#	fail2ban/server/server.py
---
 bin/fail2ban-client               | 117 ++++++++++++++++++++++--------
 bin/fail2ban-server               |   3 +-
 fail2ban/client/fail2banreader.py |   9 ++-
 fail2ban/server/server.py         |  18 ++---
 4 files changed, 106 insertions(+), 41 deletions(-)

diff --git a/bin/fail2ban-client b/bin/fail2ban-client
index bc0c0be8..631afb31 100755
--- a/bin/fail2ban-client
+++ b/bin/fail2ban-client
@@ -54,6 +54,7 @@ class Fail2banClient:
 	PROMPT = "fail2ban> "
 
 	def __init__(self):
+		self.__server = None
 		self.__argv = None
 		self.__stream = None
 		self.__configurator = Configurator()
@@ -89,13 +90,16 @@ class Fail2banClient:
 		print "    -c <DIR>                configuration directory"
 		print "    -s <FILE>               socket path"
 		print "    -p <FILE>               pidfile path"
+		print "    --loglevel <LEVEL>      logging level"
+		print "    --logtarget <FILE>|STDOUT|STDERR|SYSLOG"
+		print "    --syslogsocket auto|file"
 		print "    -d                      dump configuration. For debugging"
 		print "    -i                      interactive mode"
 		print "    -v                      increase verbosity"
 		print "    -q                      decrease verbosity"
 		print "    -x                      force execution of the server (remove socket file)"
 		print "    -b                      start server in background (default)"
-		print "    -f                      start server in foreground (note that the client forks once itself)"
+		print "    -f                      start server in foreground"
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print
@@ -128,6 +132,8 @@ class Fail2banClient:
 				self.__conf["socket"] = opt[1]
 			elif opt[0] == "-p":
 				self.__conf["pidfile"] = opt[1]
+			elif opt[0].startswith("--log") or opt[0].startswith("--sys"):
+				self.__conf[ opt[0][2:] ] = opt[1]
 			elif opt[0] == "-d":
 				self.__conf["dump"] = True
 			elif opt[0] == "-v":
@@ -234,24 +240,32 @@ class Fail2banClient:
 						"Directory %s exists but not accessible for writing"
 						% (socket_dir,))
 					return False
-				# Start the server
-				self.__startServerAsync(self.__conf["socket"],
-										self.__conf["pidfile"],
-										self.__conf["force"],
-										self.__conf["background"])
-				try:
-					# Wait for the server to start
-					self.__waitOnServer()
-					# Configure the server
-					self.__processCmd(self.__stream, False)
-					return True
-				except ServerExecutionException:
-					logSys.error("Could not start server. Maybe an old "
-								 "socket file is still present. Try to "
-								 "remove " + self.__conf["socket"] + ". If "
-								 "you used fail2ban-client to start the "
-								 "server, adding the -x option will do it")
+
+				# Check already running
+				if not self.__conf["force"] and os.path.exists(self.__conf["socket"]):
+					logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)")
 					return False
+
+				# Start the server
+				t = None
+				if self.__conf["background"]:
+					# Start server daemon as fork of client process:
+					self.__startServerAsync()
+					# Send config stream to server:
+					return self.__processStartStreamAfterWait()
+				else:
+					# In foreground mode we should start server/client communication in other thread:
+					from threading import Thread
+					t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,))
+					t.start()
+					# Start server direct here in main thread:
+					try:
+						self.__startServerDirect()
+					except KeyboardInterrupt:
+						None
+
+				return True
+
 		elif len(cmd) == 1 and cmd[0] == "reload":
 			if self.__ping():
 				ret = self.__readConfig()
@@ -281,12 +295,50 @@ class Fail2banClient:
 			return self.__processCmd([cmd])
 
 
+	def __processStartStreamAfterWait(self):
+		try:
+			# Wait for the server to start
+			self.__waitOnServer()
+			# Configure the server
+			self.__processCmd(self.__stream, False)
+		except ServerExecutionException:
+			logSys.error("Could not start server. Maybe an old "
+						 "socket file is still present. Try to "
+						 "remove " + self.__conf["socket"] + ". If "
+						 "you used fail2ban-client to start the "
+						 "server, adding the -x option will do it")
+			if not self.__conf["background"]:
+				self.__server.quit()
+				sys.exit(-1)
+			return False
+		return True
+
+
+	##
+	# Start Fail2Ban server in main thread without fork (foreground).
+	#
+	# Start the Fail2ban server in foreground (daemon mode or not).
+
+	def __startServerDirect(self):
+		from fail2ban.server.server import Server
+		try:
+			self.__server = Server(False)
+			self.__server.start(self.__conf["socket"],
+							self.__conf["pidfile"], self.__conf["force"], 
+							conf=self.__conf)
+		except Exception, e:
+			logSys.exception(e)
+			if self.__server:
+				self.__server.quit()
+			sys.exit(-1)
+
+
 	##
 	# Start Fail2Ban server.
 	#
 	# Start the Fail2ban server in daemon mode.
 
-	def __startServerAsync(self, socket, pidfile, force = False, background = True):
+	def __startServerAsync(self):
 		# Forks the current process.
 		pid = os.fork()
 		if pid == 0:
@@ -294,18 +346,15 @@ class Fail2banClient:
 			args.append(self.SERVER)
 			# Set the socket path.
 			args.append("-s")
-			args.append(socket)
+			args.append(self.__conf["socket"])
 			# Set the pidfile
 			args.append("-p")
-			args.append(pidfile)
+			args.append(self.__conf["pidfile"])
 			# Force the execution if needed.
-			if force:
+			if self.__conf["force"]:
 				args.append("-x")
-			# Start in foreground mode if requested.
-			if background:
-				args.append("-b")
-			else:
-				args.append("-f")
+			# Start in background as requested.
+			args.append("-b")
 			
 			try:
 				# Use the current directory.
@@ -361,7 +410,7 @@ class Fail2banClient:
 		# Reads the command line options.
 		try:
 			cmdOpts = 'hc:s:p:xfbdviqV'
-			cmdLongOpts = ['help', 'version']
+			cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version']
 			optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts)
 		except getopt.GetoptError:
 			self.dispUsage()
@@ -398,7 +447,17 @@ class Fail2banClient:
 			self.__conf["socket"] = conf["socket"]
 		if self.__conf["pidfile"] is None:
 			self.__conf["pidfile"] = conf["pidfile"]
-		logSys.info("Using socket file " + self.__conf["socket"])
+		if self.__conf.get("logtarget", None) is None:
+			self.__conf["logtarget"] = conf["logtarget"]
+		if self.__conf.get("loglevel", None) is None:
+			self.__conf["loglevel"] = conf["loglevel"]
+		if self.__conf.get("syslogsocket", None) is None:
+			self.__conf["syslogsocket"] = conf["syslogsocket"]
+
+		logSys.info("Using socket file %s", self.__conf["socket"])
+
+		logSys.info("Using pid file %s, [%s] logging to %s",
+			self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"])
 
 		if self.__conf["dump"]:
 			ret = self.__readConfig()
diff --git a/bin/fail2ban-server b/bin/fail2ban-server
index f522f418..0b8b6418 100755
--- a/bin/fail2ban-server
+++ b/bin/fail2ban-server
@@ -129,7 +129,8 @@ class Fail2banServer:
 			return True
 		except Exception, e:
 			logSys.exception(e)
-			self.__server.quit()
+			if self.__server:
+				self.__server.quit()
 			return False
 
 if __name__ == "__main__":
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index c55f65ea..b3012c9b 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -40,8 +40,13 @@ class Fail2banReader(ConfigReader):
 		ConfigReader.read(self, "fail2ban")
 	
 	def getEarlyOptions(self):
-		opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"],
-				["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]]
+		opts = [
+			["string", "socket", "/var/run/fail2ban/fail2ban.sock"],
+			["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"],
+			["string", "loglevel", "INFO"],
+			["string", "logtarget", "/var/log/fail2ban.log"],
+			["string", "syslogsocket", "auto"]
+		]
 		return ConfigReader.getOptions(self, "Definition", opts)
 	
 	def getOptions(self):
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 7f75c347..1d3e89d3 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -67,10 +67,6 @@ class Server:
 			'FreeBSD': '/var/run/log',
 			'Linux': '/dev/log',
 		}
-		self.setSyslogSocket("auto")
-		# Set logging level
-		self.setLogLevel("INFO")
-		self.setLogTarget("STDOUT")
 
 	def __sigTERMhandler(self, signum, frame):
 		logSys.debug("Caught signal %d. Exiting" % signum)
@@ -80,7 +76,12 @@ class Server:
 		logSys.debug("Caught signal %d. Flushing logs" % signum)
 		self.flushLogs()
 
-	def start(self, sock, pidfile, force = False):
+	def start(self, sock, pidfile, force=False, conf={}):
+		# First set all logging parameters:
+		self.setSyslogSocket(conf.get("syslogsocket", "auto"))
+		self.setLogLevel(conf.get("loglevel", "INFO"))
+		self.setLogTarget(conf.get("logtarget", "STDOUT"))
+
 		logSys.info("Starting Fail2ban v%s", version.version)
 		
 		# Install signal handlers
@@ -402,8 +403,9 @@ class Server:
 	# @param target the logging target
 	
 	def setLogTarget(self, target):
-		try:
-			self.__loggingLock.acquire()
+		with self.__loggingLock:
+			if self.__logTarget == target:
+				return True
 			# set a format which is simpler for console use
 			formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s")
 			if target == "SYSLOG":
@@ -471,8 +473,6 @@ class Server:
 			# Sets the logging target.
 			self.__logTarget = target
 			return True
-		finally:
-			self.__loggingLock.release()
 
 	##
 	# Sets the syslog socket.

From 556ddaabd7a0b1f1ac94d014680fa83d980f770f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Feb 2016 21:00:00 +0100
Subject: [PATCH 073/623] temporary commit (move client/server from bin)

---
 bin/fail2ban-client => fail2ban/client/fail2banclient.py | 0
 bin/fail2ban-server => fail2ban/client/fail2banserver.py | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename bin/fail2ban-client => fail2ban/client/fail2banclient.py (100%)
 rename bin/fail2ban-server => fail2ban/client/fail2banserver.py (100%)

diff --git a/bin/fail2ban-client b/fail2ban/client/fail2banclient.py
similarity index 100%
rename from bin/fail2ban-client
rename to fail2ban/client/fail2banclient.py
diff --git a/bin/fail2ban-server b/fail2ban/client/fail2banserver.py
similarity index 100%
rename from bin/fail2ban-server
rename to fail2ban/client/fail2banserver.py

From 5a053f4b743ca482bc720c5e851c5a793cad0089 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Feb 2016 08:56:12 +0100
Subject: [PATCH 074/623] starting of the server (and client/server
 communication behavior during start and daemonize) completely rewritten:   -
 client/server functionality moved away from bin and using now the common
 interface (introduced in fail2bancmdline);   - start in foreground fixed;   -
 server can act as client corresponding command line;   - command "restart"
 added: in opposite to "reload" in reality restarts the server (new process); 
  - several client/server bugs during starting process fixed.

---
 MANIFEST                           |   3 +
 bin/fail2ban-client                |  37 ++
 bin/fail2ban-server                |  37 ++
 fail2ban/client/beautifier.py      |   2 +
 fail2ban/client/fail2banclient.py  | 580 +++++++++++------------------
 fail2ban/client/fail2bancmdline.py | 247 ++++++++++++
 fail2ban/client/fail2banserver.py  | 234 +++++++-----
 fail2ban/protocol.py               |   6 +-
 fail2ban/server/server.py          |  95 ++---
 fail2ban/server/transmitter.py     |   2 +
 10 files changed, 745 insertions(+), 498 deletions(-)
 create mode 100755 bin/fail2ban-client
 create mode 100755 bin/fail2ban-server
 create mode 100644 fail2ban/client/fail2bancmdline.py
 mode change 100755 => 100644 fail2ban/client/fail2banserver.py

diff --git a/MANIFEST b/MANIFEST
index 11f635c3..713b64c3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -156,8 +156,11 @@ fail2ban/client/configparserinc.py
 fail2ban/client/configreader.py
 fail2ban/client/configurator.py
 fail2ban/client/csocket.py
+fail2ban/client/fail2banclient.py
+fail2ban/client/fail2bancmdline.py
 fail2ban/client/fail2banreader.py
 fail2ban/client/fail2banregex.py
+fail2ban/client/fail2banserver.py
 fail2ban/client/filterreader.py
 fail2ban/client/__init__.py
 fail2ban/client/jailreader.py
diff --git a/bin/fail2ban-client b/bin/fail2ban-client
new file mode 100755
index 00000000..19e76a98
--- /dev/null
+++ b/bin/fail2ban-client
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""
+Fail2Ban reads log file that contains password failure report
+and bans the corresponding IP addresses using firewall rules.
+
+This tools starts/stops fail2ban server or does client/server communication,
+to change/read parameters of the server or jails.
+
+"""
+
+__author__ = "Fail2Ban Developers"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
+__license__ = "GPL"
+
+from fail2ban.client.fail2banclient import exec_command_line
+
+if __name__ == "__main__":
+	exec_command_line()
diff --git a/bin/fail2ban-server b/bin/fail2ban-server
new file mode 100755
index 00000000..8e64d865
--- /dev/null
+++ b/bin/fail2ban-server
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""
+Fail2Ban reads log file that contains password failure report
+and bans the corresponding IP addresses using firewall rules.
+
+This tools starts/stops fail2ban server or does client/server communication,
+to change/read parameters of the server or jails.
+
+"""
+
+__author__ = "Fail2Ban Developers"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
+__license__ = "GPL"
+
+from fail2ban.client.fail2banserver import exec_command_line
+
+if __name__ == "__main__":
+	exec_command_line()
diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 7b27e92f..bd803a6a 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -68,6 +68,8 @@ class Beautifier:
 				msg = "Added jail " + response
 			elif inC[0] == "flushlogs":
 				msg = "logs: " + response
+			elif inC[0] == "echo":
+				msg = ' '.join(msg)
 			elif inC[0:1] == ['status']:
 				if len(inC) > 1:
 					# Display information
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 631afb31..7adbab95 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
-
+#
 # This file is part of Fail2Ban.
 #
 # Fail2Ban is free software; you can redistribute it and/or modify
@@ -17,99 +17,38 @@
 # You should have received a copy of the GNU General Public License
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-__author__ = "Cyril Jaquier"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
+__author__ = "Fail2Ban Developers"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
 __license__ = "GPL"
 
-import getopt
-import logging
 import os
-import pickle
-import re
 import shlex
 import signal
 import socket
-import string
 import sys
 import time
 
-from fail2ban.version import version
-from fail2ban.protocol import printFormatted
-from fail2ban.client.csocket import CSocket
-from fail2ban.client.configurator import Configurator
-from fail2ban.client.beautifier import Beautifier
-from fail2ban.helpers import getLogger
+from threading import Thread
 
-# Gets the instance of the logger.
-logSys = getLogger("fail2ban")
+from ..version import version
+from .csocket import CSocket
+from .beautifier import Beautifier
+from .fail2bancmdline import Fail2banCmdLine, logSys, exit
 
 ##
 #
 # @todo This class needs cleanup.
 
-class Fail2banClient:
+class Fail2banClient(Fail2banCmdLine, Thread):
 
-	SERVER = "fail2ban-server"
 	PROMPT = "fail2ban> "
 
 	def __init__(self):
-		self.__server = None
-		self.__argv = None
-		self.__stream = None
-		self.__configurator = Configurator()
-		self.__conf = dict()
-		self.__conf["conf"] = "/etc/fail2ban"
-		self.__conf["dump"] = False
-		self.__conf["force"] = False
-		self.__conf["background"] = True
-		self.__conf["verbose"] = 1
-		self.__conf["interactive"] = False
-		self.__conf["socket"] = None
-		self.__conf["pidfile"] = None
-
-	def dispVersion(self):
-		print "Fail2Ban v" + version
-		print
-		print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors"
-		print "Copyright of modifications held by their respective authors."
-		print "Licensed under the GNU General Public License v2 (GPL)."
-		print
-		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
-		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
-
-	def dispUsage(self):
-		""" Prints Fail2Ban command line options and exits
-		"""
-		print "Usage: "+self.__argv[0]+" [OPTIONS] <COMMAND>"
-		print
-		print "Fail2Ban v" + version + " reads log file that contains password failure report"
-		print "and bans the corresponding IP addresses using firewall rules."
-		print
-		print "Options:"
-		print "    -c <DIR>                configuration directory"
-		print "    -s <FILE>               socket path"
-		print "    -p <FILE>               pidfile path"
-		print "    --loglevel <LEVEL>      logging level"
-		print "    --logtarget <FILE>|STDOUT|STDERR|SYSLOG"
-		print "    --syslogsocket auto|file"
-		print "    -d                      dump configuration. For debugging"
-		print "    -i                      interactive mode"
-		print "    -v                      increase verbosity"
-		print "    -q                      decrease verbosity"
-		print "    -x                      force execution of the server (remove socket file)"
-		print "    -b                      start server in background (default)"
-		print "    -f                      start server in foreground"
-		print "    -h, --help              display this help message"
-		print "    -V, --version           print the version"
-		print
-		print "Command:"
-
-		# Prints the protocol
-		printFormatted()
-
-		print
-		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
+		Fail2banCmdLine.__init__(self)
+		Thread.__init__(self)
+		self._alive = True
+		self._server = None
+		self._beautifier = None
 
 	def dispInteractive(self):
 		print "Fail2Ban v" + version + " reads log file that contains password failure report"
@@ -120,58 +59,32 @@ class Fail2banClient:
 		# Print a new line because we probably come from wait
 		print
 		logSys.warning("Caught signal %d. Exiting" % signum)
-		sys.exit(-1)
-
-	def __getCmdLineOptions(self, optList):
-		""" Gets the command line options
-		"""
-		for opt in optList:
-			if opt[0] == "-c":
-				self.__conf["conf"] = opt[1]
-			elif opt[0] == "-s":
-				self.__conf["socket"] = opt[1]
-			elif opt[0] == "-p":
-				self.__conf["pidfile"] = opt[1]
-			elif opt[0].startswith("--log") or opt[0].startswith("--sys"):
-				self.__conf[ opt[0][2:] ] = opt[1]
-			elif opt[0] == "-d":
-				self.__conf["dump"] = True
-			elif opt[0] == "-v":
-				self.__conf["verbose"] = self.__conf["verbose"] + 1
-			elif opt[0] == "-q":
-				self.__conf["verbose"] = self.__conf["verbose"] - 1
-			elif opt[0] == "-x":
-				self.__conf["force"] = True
-			elif opt[0] == "-i":
-				self.__conf["interactive"] = True
-			elif opt[0] == "-b":
-				self.__conf["background"] = True
-			elif opt[0] == "-f":
-				self.__conf["background"] = False
-			elif opt[0] in ["-h", "--help"]:
-				self.dispUsage()
-				sys.exit(0)
-			elif opt[0] in ["-V", "--version"]:
-				self.dispVersion()
-				sys.exit(0)
+		exit(-1)
 
 	def __ping(self):
 		return self.__processCmd([["ping"]], False)
 
-	def __processCmd(self, cmd, showRet = True):
+	@property
+	def beautifier(self):
+		if self._beautifier:
+			return self._beautifier
+		self._beautifier = Beautifier()
+		return self._beautifier
+
+	def __processCmd(self, cmd, showRet=True):
 		client = None
 		try:
-			beautifier = Beautifier()
+			beautifier = self.beautifier
 			streamRet = True
 			for c in cmd:
 				beautifier.setInputCmd(c)
 				try:
 					if not client:
-						client = CSocket(self.__conf["socket"])
+						client = CSocket(self._conf["socket"])
 					ret = client.send(c)
 					if ret[0] == 0:
 						logSys.debug("OK : " + `ret[1]`)
-						if showRet:
+						if showRet or c[0] == 'echo':
 							print beautifier.beautify(ret[1])
 					else:
 						logSys.error("NOK: " + `ret[1].args`)
@@ -179,38 +92,126 @@ class Fail2banClient:
 							print beautifier.beautifyError(ret[1])
 						streamRet = False
 				except socket.error:
-					if showRet:
+					if showRet or self._conf["verbose"] > 1:
 						self.__logSocketError()
 					return False
 				except Exception, e:
-					if showRet:
+					if showRet or self._conf["verbose"] > 1:
 						logSys.error(e)
 					return False
 		finally:
 			if client:
 				client.close()
+			if showRet or c[0] == 'echo':
+				sys.stdout.flush()
 		return streamRet
 
 	def __logSocketError(self):
 		try:
-			if os.access(self.__conf["socket"], os.F_OK):
+			if os.access(self._conf["socket"], os.F_OK):
 				# This doesn't check if path is a socket,
 				#  but socket.error should be raised
-				if os.access(self.__conf["socket"], os.W_OK):
+				if os.access(self._conf["socket"], os.W_OK):
 					# Permissions look good, but socket.error was raised
 					logSys.error("Unable to contact server. Is it running?")
 				else:
 					logSys.error("Permission denied to socket: %s,"
-								 " (you must be root)", self.__conf["socket"])
+								 " (you must be root)", self._conf["socket"])
 			else:
 				logSys.error("Failed to access socket path: %s."
 							 " Is fail2ban running?",
-							 self.__conf["socket"])
+							 self._conf["socket"])
 		except Exception as e:
 			logSys.error("Exception while checking socket access: %s",
-						 self.__conf["socket"])
+						 self._conf["socket"])
 			logSys.error(e)
 
+	##
+	def __prepareStartServer(self):
+		if self.__ping():
+			logSys.error("Server already running")
+			return None
+
+		# Read the config
+		ret, stream = self.readConfig()
+		# Do not continue if configuration is not 100% valid
+		if not ret:
+			return None
+
+		# verify that directory for the socket file exists
+		socket_dir = os.path.dirname(self._conf["socket"])
+		if not os.path.exists(socket_dir):
+			logSys.error(
+				"There is no directory %s to contain the socket file %s."
+				% (socket_dir, self._conf["socket"]))
+			return None
+		if not os.access(socket_dir, os.W_OK | os.X_OK):
+			logSys.error(
+				"Directory %s exists but not accessible for writing"
+				% (socket_dir,))
+			return None
+
+		# Check already running
+		if not self._conf["force"] and os.path.exists(self._conf["socket"]):
+			logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)")
+			return None
+
+		stream.append(['echo', 'Server ready'])
+		return stream
+
+	##
+	def __startServer(self, background=True):
+		from .fail2banserver import Fail2banServer
+		stream = self.__prepareStartServer()
+		self._alive = True
+		if not stream:
+			return False
+		# Start the server or just initialize started one:
+		try:
+			if background:
+				# Start server daemon as fork of client process:
+				Fail2banServer.startServerAsync(self._conf)
+				# Send config stream to server:
+				if not self.__processStartStreamAfterWait(stream, False):
+					return False
+			else:
+				# In foreground mode we should make server/client communication in different threads:
+				Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start()
+				# Mark current (main) thread as daemon:
+				self.setDaemon(True)
+				# Start server direct here in main thread (not fork):
+				self._server = Fail2banServer.startServerDirect(self._conf, False)
+
+		except Exception as e:
+			print
+			logSys.error("Exception while starting server foreground")
+			logSys.error(e)
+		finally:
+			self._alive = False
+
+		return True
+
+	##
+	def configureServer(self, async=True, phase=None):
+		# if asynchron start this operation in the new thread:
+		if async:
+			return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start()
+		# prepare: read config, check configuration is valid, etc.:
+		if phase is not None:
+			phase['start'] = True
+			logSys.debug('-- client phase %s', phase)
+		stream = self.__prepareStartServer()
+		if phase is not None:
+			phase['ready'] = phase['start'] = (True if stream else False)
+			logSys.debug('-- client phase %s', phase)
+		if not stream:
+			return False
+		# configure server with config stream:
+		ret = self.__processStartStreamAfterWait(stream, False)
+		if phase is not None:
+			phase['done'] = ret
+		return ret
+
 	##
 	# Process a command line.
 	#
@@ -219,253 +220,101 @@ class Fail2banClient:
 
 	def __processCommand(self, cmd):
 		if len(cmd) == 1 and cmd[0] == "start":
-			if self.__ping():
-				logSys.error("Server already running")
+			
+			ret = self.__startServer(self._conf["background"])
+			if not ret:
 				return False
-			else:
-				# Read the config
-				ret = self.__readConfig()
-				# Do not continue if configuration is not 100% valid
-				if not ret:
-					return False
-				# verify that directory for the socket file exists
-				socket_dir = os.path.dirname(self.__conf["socket"])
-				if not os.path.exists(socket_dir):
-					logSys.error(
-						"There is no directory %s to contain the socket file %s."
-						% (socket_dir, self.__conf["socket"]))
-					return False
-				if not os.access(socket_dir, os.W_OK | os.X_OK):
-					logSys.error(
-						"Directory %s exists but not accessible for writing"
-						% (socket_dir,))
-					return False
+			return ret
 
-				# Check already running
-				if not self.__conf["force"] and os.path.exists(self.__conf["socket"]):
-					logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)")
-					return False
+		elif len(cmd) == 1 and cmd[0] == "restart":
 
-				# Start the server
-				t = None
-				if self.__conf["background"]:
-					# Start server daemon as fork of client process:
-					self.__startServerAsync()
-					# Send config stream to server:
-					return self.__processStartStreamAfterWait()
+			if self._conf.get("interactive", False):
+				print('  ## stop ... ')
+			self.__processCommand(['stop'])
+			self.__waitOnServer(False)
+			# in interactive mode reset config, to make full-reload if there something changed:
+			if self._conf.get("interactive", False):
+				print('  ## load configuration ... ')
+				self.resetConf()
+				ret = self.initCmdLine(self._argv)
+				if ret is not None:
+					return ret
+			if self._conf.get("interactive", False):
+				print('  ## start ... ')
+			return self.__processCommand(['start'])
+
+		elif len(cmd) >= 1 and cmd[0] == "reload":
+			if self.__ping():
+				if len(cmd) == 1:
+					jail = 'all'
+					ret, stream = self.readConfig()
 				else:
-					# In foreground mode we should start server/client communication in other thread:
-					from threading import Thread
-					t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,))
-					t.start()
-					# Start server direct here in main thread:
-					try:
-						self.__startServerDirect()
-					except KeyboardInterrupt:
-						None
-
-				return True
-
-		elif len(cmd) == 1 and cmd[0] == "reload":
-			if self.__ping():
-				ret = self.__readConfig()
-				# Do not continue if configuration is not 100% valid
-				if not ret:
-					return False
-				self.__processCmd([['stop', 'all']], False)
-				# Configure the server
-				return self.__processCmd(self.__stream, False)
-			else:
-				logSys.error("Could not find server")
-				return False
-		elif len(cmd) == 2 and cmd[0] == "reload":
-			if self.__ping():
-				jail = cmd[1]
-				ret = self.__readConfig(jail)
+					jail = cmd[1]
+					ret, stream = self.readConfig(jail)
 				# Do not continue if configuration is not 100% valid
 				if not ret:
 					return False
 				self.__processCmd([['stop', jail]], False)
 				# Configure the server
-				return self.__processCmd(self.__stream, False)
+				return self.__processCmd(stream, True)
 			else:
 				logSys.error("Could not find server")
 				return False
+
 		else:
 			return self.__processCmd([cmd])
 
 
-	def __processStartStreamAfterWait(self):
+	def __processStartStreamAfterWait(self, *args):
 		try:
 			# Wait for the server to start
 			self.__waitOnServer()
-			# Configure the server
-			self.__processCmd(self.__stream, False)
+				# Configure the server
+			self.__processCmd(*args)
 		except ServerExecutionException:
 			logSys.error("Could not start server. Maybe an old "
 						 "socket file is still present. Try to "
-						 "remove " + self.__conf["socket"] + ". If "
+						 "remove " + self._conf["socket"] + ". If "
 						 "you used fail2ban-client to start the "
 						 "server, adding the -x option will do it")
-			if not self.__conf["background"]:
-				self.__server.quit()
-				sys.exit(-1)
+			if self._server:
+				self._server.quit()
+				exit(-1)
 			return False
 		return True
 
-
-	##
-	# Start Fail2Ban server in main thread without fork (foreground).
-	#
-	# Start the Fail2ban server in foreground (daemon mode or not).
-
-	def __startServerDirect(self):
-		from fail2ban.server.server import Server
-		try:
-			self.__server = Server(False)
-			self.__server.start(self.__conf["socket"],
-							self.__conf["pidfile"], self.__conf["force"], 
-							conf=self.__conf)
-		except Exception, e:
-			logSys.exception(e)
-			if self.__server:
-				self.__server.quit()
-			sys.exit(-1)
-
-
-	##
-	# Start Fail2Ban server.
-	#
-	# Start the Fail2ban server in daemon mode.
-
-	def __startServerAsync(self):
-		# Forks the current process.
-		pid = os.fork()
-		if pid == 0:
-			args = list()
-			args.append(self.SERVER)
-			# Set the socket path.
-			args.append("-s")
-			args.append(self.__conf["socket"])
-			# Set the pidfile
-			args.append("-p")
-			args.append(self.__conf["pidfile"])
-			# Force the execution if needed.
-			if self.__conf["force"]:
-				args.append("-x")
-			# Start in background as requested.
-			args.append("-b")
-			
-			try:
-				# Use the current directory.
-				exe = os.path.abspath(os.path.join(sys.path[0], self.SERVER))
-				logSys.debug("Starting %r with args %r" % (exe, args))
-				os.execv(exe, args)
-			except OSError:
-				try:
-					# Use the PATH env.
-					logSys.warning("Initial start attempt failed.  Starting %r with the same args" % (self.SERVER,))
-					os.execvp(self.SERVER, args)
-				except OSError:
-					logSys.error("Could not start %s" % self.SERVER)
-					os.exit(-1)
-
-	def __waitOnServer(self):
-		# Wait for the server to start
-		cnt = 0
-		if self.__conf["verbose"] > 1:
-			pos = 0
-			delta = 1
-			mask = "[          ]"
-		while not self.__ping():
-			# Wonderful visual :)
-			if self.__conf["verbose"] > 1:
-				pos += delta
-				sys.stdout.write("\rINFO   " + mask[:pos] + '#' + mask[pos+1:] +
-								 " Waiting on the server...")
-				sys.stdout.flush()
-				if pos > len(mask)-3:
-					delta = -1
-				elif pos < 2:
-					delta = 1
-			# The server has 30 seconds to start.
-			if cnt >= 300:
-				if self.__conf["verbose"] > 1:
-					sys.stdout.write('\n')
-				raise ServerExecutionException("Failed to start server")
-			time.sleep(0.1)
-			cnt += 1
-		if self.__conf["verbose"] > 1:
-			sys.stdout.write('\n')
-
+	def __waitOnServer(self, alive=True, maxtime=30):
+		# Wait for the server to start (the server has 30 seconds to answer ping)
+		starttime = time.time()
+		with VisualWait(self._conf["verbose"]) as vis:
+			while self._alive and not self.__ping() == alive or (
+				not alive and os.path.exists(self._conf["socket"])
+			):
+				now = time.time()
+				# Wonderful visual :)
+				if now > starttime + 1:
+					vis.heartbeat()
+				# f end time reached:
+				if now - starttime >= maxtime:
+					raise ServerExecutionException("Failed to start server")
+				time.sleep(0.1)
 
 	def start(self, argv):
-		# Command line options
-		self.__argv = argv
-
 		# Install signal handlers
 		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
 		signal.signal(signal.SIGINT, self.__sigTERMhandler)
 
-		# Reads the command line options.
-		try:
-			cmdOpts = 'hc:s:p:xfbdviqV'
-			cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version']
-			optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts)
-		except getopt.GetoptError:
-			self.dispUsage()
-			return False
+		# Command line options
+		if self._argv is None:
+			ret = self.initCmdLine(argv)
+			if ret is not None:
+				return ret
 
-		self.__getCmdLineOptions(optList)
-
-		verbose = self.__conf["verbose"]
-		if verbose <= 0:
-			logSys.setLevel(logging.ERROR)
-		elif verbose == 1:
-			logSys.setLevel(logging.WARNING)
-		elif verbose == 2:
-			logSys.setLevel(logging.INFO)
-		elif verbose == 3:
-			logSys.setLevel(logging.DEBUG)
-		else:
-			logSys.setLevel(logging.HEAVYDEBUG)
-		# Add the default logging handler to dump to stderr
-		logout = logging.StreamHandler(sys.stderr)
-		# set a format which is simpler for console use
-		formatter = logging.Formatter('%(levelname)-6s %(message)s')
-		# tell the handler to use this format
-		logout.setFormatter(formatter)
-		logSys.addHandler(logout)
-
-		# Set the configuration path
-		self.__configurator.setBaseDir(self.__conf["conf"])
-
-		# Set socket path
-		self.__configurator.readEarly()
-		conf = self.__configurator.getEarlyOptions()
-		if self.__conf["socket"] is None:
-			self.__conf["socket"] = conf["socket"]
-		if self.__conf["pidfile"] is None:
-			self.__conf["pidfile"] = conf["pidfile"]
-		if self.__conf.get("logtarget", None) is None:
-			self.__conf["logtarget"] = conf["logtarget"]
-		if self.__conf.get("loglevel", None) is None:
-			self.__conf["loglevel"] = conf["loglevel"]
-		if self.__conf.get("syslogsocket", None) is None:
-			self.__conf["syslogsocket"] = conf["syslogsocket"]
-
-		logSys.info("Using socket file %s", self.__conf["socket"])
-
-		logSys.info("Using pid file %s, [%s] logging to %s",
-			self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"])
-
-		if self.__conf["dump"]:
-			ret = self.__readConfig()
-			self.dumpConfig(self.__stream)
-			return ret
+		# Commands
+		args = self._args
 
 		# Interactive mode
-		if self.__conf["interactive"]:
+		if self._conf.get("interactive", False):
 			try:
 				import readline
 			except ImportError:
@@ -500,35 +349,56 @@ class Fail2banClient:
 				return False
 			return self.__processCommand(args)
 
-	def __readConfig(self, jail=None):
-		# Read the configuration
-		# TODO: get away from stew of return codes and exception
-		# handling -- handle via exceptions
-		try:
-			self.__configurator.Reload()
-			self.__configurator.readAll()
-			ret = self.__configurator.getOptions(jail)
-			self.__configurator.convertToProtocol()
-			self.__stream = self.__configurator.getConfigStream()
-		except Exception, e:
-			logSys.error("Failed during configuration: %s" % e)
-			ret = False
-		return ret
-
-	@staticmethod
-	def dumpConfig(cmd):
-		for c in cmd:
-			print c
-		return True
-
 
 class ServerExecutionException(Exception):
 	pass
 
-if __name__ == "__main__": # pragma: no cover - can't test main
+
+##
+# Wonderful visual :)
+#
+
+class _VisualWait:
+	pos = 0
+	delta = 1
+	maxpos = 10
+	def __enter__(self):
+		return self
+	def __exit__(self, *args):
+		if self.pos:
+			sys.stdout.write('\r'+(' '*(35+self.maxpos))+'\r')
+			sys.stdout.flush()
+	def heartbeat(self):
+		if not self.pos:
+			sys.stdout.write("\nINFO   [#" + (' '*self.maxpos) + "] Waiting on the server...\r\x1b[8C")
+		self.pos += self.delta
+		if self.delta > 0:
+			s = " #\x1b[1D" if self.pos > 1 else "# \x1b[2D"
+		else: 
+			s = "\x1b[1D# \x1b[2D"
+		sys.stdout.write(s)
+		sys.stdout.flush()
+		if self.pos > self.maxpos:
+			self.delta = -1
+		elif self.pos < 2:
+			self.delta = 1
+class _NotVisualWait:
+	def __enter__(self):
+		return self
+	def __exit__(self, *args):
+		pass
+	def heartbeat(self):
+		pass
+
+def VisualWait(verbose):
+	return _VisualWait() if verbose > 1 else _NotVisualWait()
+
+
+def exec_command_line(): # pragma: no cover - can't test main
 	client = Fail2banClient()
 	# Exit with correct return value
 	if client.start(sys.argv):
-		sys.exit(0)
+		exit(0)
 	else:
-		sys.exit(-1)
+		exit(-1)
+
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
new file mode 100644
index 00000000..e23f9b19
--- /dev/null
+++ b/fail2ban/client/fail2bancmdline.py
@@ -0,0 +1,247 @@
+#!/usr/bin/python
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+#
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+__author__ = "Fail2Ban Developers"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
+__license__ = "GPL"
+
+import getopt
+import logging
+import os
+import sys
+
+from ..version import version
+from ..protocol import printFormatted
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger("fail2ban")
+
+CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",)
+
+
+class Fail2banCmdLine():
+
+	def __init__(self):
+		self._argv = self._args = None
+		self._configurator = None
+		self.resetConf()
+
+	def resetConf(self):
+		self._conf = {
+		  "async": False,
+			"conf": "/etc/fail2ban",
+			"force": False,
+			"background": True,
+			"verbose": 1,
+			"socket": None,
+			"pidfile": None
+		}
+
+	@property
+	def configurator(self):
+		if self._configurator:
+			return self._configurator
+		# New configurator
+		from .configurator import Configurator
+		self._configurator = Configurator()
+		# Set the configuration path
+		self._configurator.setBaseDir(self._conf["conf"])
+		return self._configurator
+
+
+	def applyMembers(self, obj):
+		for o in obj.__dict__:
+			self.__dict__[o] = obj.__dict__[o]
+
+	def dispVersion(self):
+		print "Fail2Ban v" + version
+		print
+		print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors"
+		print "Copyright of modifications held by their respective authors."
+		print "Licensed under the GNU General Public License v2 (GPL)."
+		print
+		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
+		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
+
+	def dispUsage(self):
+		""" Prints Fail2Ban command line options and exits
+		"""
+		caller = os.path.basename(self._argv[0])
+		print "Usage: "+caller+" [OPTIONS]" + (" <COMMAND>" if not caller.endswith('server') else "")
+		print
+		print "Fail2Ban v" + version + " reads log file that contains password failure report"
+		print "and bans the corresponding IP addresses using firewall rules."
+		print
+		print "Options:"
+		print "    -c <DIR>                configuration directory"
+		print "    -s <FILE>               socket path"
+		print "    -p <FILE>               pidfile path"
+		print "    --loglevel <LEVEL>      logging level"
+		print "    --logtarget <FILE>|STDOUT|STDERR|SYSLOG"
+		print "    --syslogsocket auto|<FILE>"
+		print "    -d                      dump configuration. For debugging"
+		print "    -i                      interactive mode"
+		print "    -v                      increase verbosity"
+		print "    -q                      decrease verbosity"
+		print "    -x                      force execution of the server (remove socket file)"
+		print "    -b                      start server in background (default)"
+		print "    -f                      start server in foreground"
+		print "    --async                 start server in async mode (for internal usage only, don't read configuration)"
+		print "    -h, --help              display this help message"
+		print "    -V, --version           print the version"
+		
+		if not caller.endswith('server'):
+			print
+			print "Command:"
+			# Prints the protocol
+			printFormatted()
+
+		print
+		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
+
+	def __getCmdLineOptions(self, optList):
+		""" Gets the command line options
+		"""
+		for opt in optList:
+			o = opt[0]
+			if o == "--async":
+				self._conf["async"] = True
+			if o == "-c":
+				self._conf["conf"] = opt[1]
+			elif o == "-s":
+				self._conf["socket"] = opt[1]
+			elif o == "-p":
+				self._conf["pidfile"] = opt[1]
+			elif o.startswith("--log") or o.startswith("--sys"):
+				self._conf[ o[2:] ] = opt[1]
+			elif o == "-d":
+				self._conf["dump"] = True
+			elif o == "-v":
+				self._conf["verbose"] += 1
+			elif o == "-q":
+				self._conf["verbose"] -= 1
+			elif o == "-x":
+				self._conf["force"] = True
+			elif o == "-i":
+				self._conf["interactive"] = True
+			elif o == "-b":
+				self._conf["background"] = True
+			elif o == "-f":
+				self._conf["background"] = False
+			elif o in ["-h", "--help"]:
+				self.dispUsage()
+				exit(0)
+			elif o in ["-V", "--version"]:
+				self.dispVersion()
+				exit(0)
+
+	def initCmdLine(self, argv):
+		# First time?
+		initial = (self._argv is None)
+
+		# Command line options
+		self._argv = argv
+
+		# Reads the command line options.
+		try:
+			cmdOpts = 'hc:s:p:xfbdviqV'
+			cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version']
+			optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
+		except getopt.GetoptError:
+			self.dispUsage()
+			exit(-1)
+
+		self.__getCmdLineOptions(optList)
+
+		if initial:
+			verbose = self._conf["verbose"]
+			if verbose <= 0:
+				logSys.setLevel(logging.ERROR)
+			elif verbose == 1:
+				logSys.setLevel(logging.WARNING)
+			elif verbose == 2:
+				logSys.setLevel(logging.INFO)
+			elif verbose == 3:
+				logSys.setLevel(logging.DEBUG)
+			else:
+				logSys.setLevel(logging.HEAVYDEBUG)
+			# Add the default logging handler to dump to stderr
+			logout = logging.StreamHandler(sys.stderr)
+			# set a format which is simpler for console use
+			formatter = logging.Formatter('%(levelname)-6s %(message)s')
+			# tell the handler to use this format
+			logout.setFormatter(formatter)
+			logSys.addHandler(logout)
+
+		# Set expected parameters (like socket, pidfile, etc) from configuration,
+		# if those not yet specified, in which read configuration only if needed here:
+		conf = None
+		for o in CONFIG_PARAMS:
+			if self._conf.get(o, None) is None:
+				if not conf:
+					self.configurator.readEarly()
+					conf = self.configurator.getEarlyOptions()
+				self._conf[o] = conf[o]
+
+		logSys.info("Using socket file %s", self._conf["socket"])
+
+		logSys.info("Using pid file %s, [%s] logging to %s",
+			self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"])
+
+		if self._conf.get("dump", False):
+			ret, stream = self.readConfig()
+			self.dumpConfig(stream)
+			return ret
+
+		# Nothing to do here, process in client/server
+		return None
+
+	def readConfig(self, jail=None):
+		# Read the configuration
+		# TODO: get away from stew of return codes and exception
+		# handling -- handle via exceptions
+		stream = None
+		try:
+			self.configurator.Reload()
+			self.configurator.readAll()
+			ret = self.configurator.getOptions(jail)
+			self.configurator.convertToProtocol()
+			stream = self.configurator.getConfigStream()
+		except Exception, e:
+			logSys.error("Failed during configuration: %s" % e)
+			ret = False
+		return ret, stream
+
+	@staticmethod
+	def dumpConfig(cmd):
+		for c in cmd:
+			print c
+		return True
+
+	@staticmethod
+	def exit(code=0):
+		logSys.debug("Exit with code %s", code)
+		if os._exit:
+			os._exit(code)
+		else:
+			sys.exit(code)
+
+# global exit handler:
+exit = Fail2banCmdLine.exit
\ No newline at end of file
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
old mode 100755
new mode 100644
index 0b8b6418..6c1dd694
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
-
+#
 # This file is part of Fail2Ban.
 #
 # Fail2Ban is free software; you can redistribute it and/or modify
@@ -17,125 +17,171 @@
 # You should have received a copy of the GNU General Public License
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-__author__ = "Cyril Jaquier"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
+__author__ = "Fail2Ban Developers"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
 __license__ = "GPL"
 
-import getopt
 import os
 import sys
 
-from fail2ban.version import version
-from fail2ban.server.server import Server
-from fail2ban.helpers import getLogger
-
-# Gets the instance of the logger.
-logSys = getLogger("fail2ban")
+from ..version import version
+from ..server.server import Server, ServerDaemonize
+from ..server.utils import Utils
+from .fail2bancmdline import Fail2banCmdLine, logSys, exit
 
+SERVER = "fail2ban-server"
 ##
 # \mainpage Fail2Ban
 #
 # \section Introduction
 #
-# Fail2ban is designed to protect your server against brute force attacks.
-# Its first goal was to protect a SSH server.
+class Fail2banServer(Fail2banCmdLine):
 
-class Fail2banServer:
+	# def __init__(self):
+	# 	Fail2banCmdLine.__init__(self)
 
-	def __init__(self):
-		self.__server = None
-		self.__argv = None
-		self.__conf = dict()
-		self.__conf["background"] = True
-		self.__conf["force"] = False
-		self.__conf["socket"] = "/var/run/fail2ban/fail2ban.sock"
-		self.__conf["pidfile"] = "/var/run/fail2ban/fail2ban.pid"
+	##
+	# Start Fail2Ban server in main thread without fork (foreground).
+	#
+	# Start the Fail2ban server in foreground (daemon mode or not).
 
-	def dispVersion(self):
-		print "Fail2Ban v" + version
-		print
-		print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors"
-		print "Copyright of modifications held by their respective authors."
-		print "Licensed under the GNU General Public License v2 (GPL)."
-		print
-		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
-		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
+	@staticmethod
+	def startServerDirect(conf, daemon=True):
+		server = None
+		try:
+			# Start it in foreground (current thread, not new process),
+			# server object will internally fork self if daemon is True
+			server = Server(daemon)
+			server.start(conf["socket"],
+							conf["pidfile"], conf["force"], 
+							conf=conf)
+		except ServerDaemonize:
+			pass
+		except Exception, e:
+			logSys.exception(e)
+			if server:
+				server.quit()
+			exit(-1)
 
-	def dispUsage(self):
-		""" Prints Fail2Ban command line options and exits
-		"""
-		print "Usage: "+self.__argv[0]+" [OPTIONS]"
-		print
-		print "Fail2Ban v" + version + " reads log file that contains password failure report"
-		print "and bans the corresponding IP addresses using firewall rules."
-		print
-		print "Only use this command for debugging purpose. Start the server with"
-		print "fail2ban-client instead. The default behaviour is to start the server"
-		print "in background."
-		print
-		print "Options:"
-		print "    -b                   start in background"
-		print "    -f                   start in foreground"
-		print "    -s <FILE>            socket path"
-		print "    -p <FILE>            pidfile path"
-		print "    -x                   force execution of the server (remove socket file)"
-		print "    -h, --help           display this help message"
-		print "    -V, --version        print the version"
-		print
-		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
+		return server
 
-	def __getCmdLineOptions(self, optList):
-		""" Gets the command line options
-		"""
-		for opt in optList:
-			if opt[0] == "-b":
-				self.__conf["background"] = True
-			if opt[0] == "-f":
-				self.__conf["background"] = False
-			if opt[0] == "-s":
-				self.__conf["socket"] = opt[1]
-			if opt[0] == "-p":
-				self.__conf["pidfile"] = opt[1]
-			if opt[0] == "-x":
-				self.__conf["force"] = True
-			if opt[0] in ["-h", "--help"]:
-				self.dispUsage()
-				sys.exit(0)
-			if opt[0] in ["-V", "--version"]:
-				self.dispVersion()
-				sys.exit(0)
+	##
+	# Start Fail2Ban server.
+	#
+	# Start the Fail2ban server in daemon mode (background, start from client).
+
+	@staticmethod
+	def startServerAsync(conf):
+		# Forks the current process.
+		pid = os.fork()
+		if pid == 0:
+			args = list()
+			args.append(SERVER)
+			# Start async (don't read config) and in background as requested.
+			args.append("--async")
+			args.append("-b")
+			# Set the socket path.
+			args.append("-s")
+			args.append(conf["socket"])
+			# Set the pidfile
+			args.append("-p")
+			args.append(conf["pidfile"])
+			# Force the execution if needed.
+			if conf["force"]:
+				args.append("-x")
+			# Logging parameters:
+			for o in ('loglevel', 'logtarget', 'syslogsocket'):
+				args.append("--"+o)
+				args.append(conf[o])
+
+			try:
+				# Use the current directory.
+				exe = os.path.abspath(os.path.join(sys.path[0], SERVER))
+				logSys.debug("Starting %r with args %r", exe, args)
+				os.execv(exe, args)
+			except OSError:
+				try:
+					# Use the PATH env.
+					logSys.warning("Initial start attempt failed.  Starting %r with the same args", SERVER)
+					os.execvp(SERVER, args)
+				except OSError:
+					exit(-1)
+
+	def _Fail2banClient(self):
+		from .fail2banclient import Fail2banClient
+		cli = Fail2banClient()
+		cli.applyMembers(self)
+		return cli
 
 	def start(self, argv):
 		# Command line options
-		self.__argv = argv
+		ret = self.initCmdLine(argv)
+		if ret is not None:
+			return ret
 
-		# Reads the command line options.
+		# Commands
+		args = self._args
+
+		cli = None
+		# If client mode - whole processing over client:
+		if len(args) or self._conf.get("interactive", False):
+			cli = self._Fail2banClient()
+			return cli.start(argv)
+
+		# Start the server:
+		server = None
 		try:
-			cmdOpts = 'bfs:p:xhV'
-			cmdLongOpts = ['help', 'version']
-			optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts)
-		except getopt.GetoptError:
-			self.dispUsage()
-			sys.exit(-1)
+			# async = True, if started from client, should fork, daemonize, etc...
+			# background = True, if should start in new process, otherwise start in foreground
+			async = self._conf.get("async", False)
+			background = self._conf["background"]
+			# If was started not from the client:
+			if not async:
+				# Start new thread with client to read configuration and
+				# transfer it to the server:
+				cli = self._Fail2banClient()
+				phase = dict()
+				logSys.debug('Configure via async client thread')
+				cli.configureServer(async=True, phase=phase)
+				# wait up to 30 seconds, do not continue if configuration is not 100% valid:
+				Utils.wait_for(lambda: phase.get('ready', None) is not None, 30)
+				if not phase.get('start', False):
+					return False
 
-		self.__getCmdLineOptions(optList)
+			# Start server, daemonize it, etc.
+			if async or not background:
+				server = Fail2banServer.startServerDirect(self._conf, background)
+			else:
+				Fail2banServer.startServerAsync(self._conf)
+			if cli:
+				cli._server = server
+
+			# wait for client answer "done":
+			if not async and cli:
+				Utils.wait_for(lambda: phase.get('done', None) is not None, 30)
+				if not phase.get('done', False):
+					if server:
+						server.quit()
+					exit(-1)
+				logSys.debug('Starting server done')
 
-		try:
-			self.__server = Server(self.__conf["background"])
-			self.__server.start(self.__conf["socket"],
-								self.__conf["pidfile"],
-								self.__conf["force"])
-			return True
 		except Exception, e:
 			logSys.exception(e)
-			if self.__server:
-				self.__server.quit()
-			return False
+			if server:
+				server.quit()
+			exit(-1)
 
-if __name__ == "__main__":
+		return True
+
+	@staticmethod
+	def exit(code=0): # pragma: no cover
+		if code != 0:
+			logSys.error("Could not start %s", SERVER)
+		exit(code)
+
+def exec_command_line(): # pragma: no cover - can't test main
 	server = Fail2banServer()
 	if server.start(sys.argv):
-		sys.exit(0)
+		exit(0)
 	else:
-		sys.exit(-1)
+		exit(-1)
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 5d9fdd65..857d5fa6 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -42,11 +42,13 @@ CSPROTO = dotdict({
 protocol = [
 ['', "BASIC", ""],
 ["start", "starts the server and the jails"], 
-["reload", "reloads the configuration"], 
+["restart", "restarts the server"], 
+["reload", "reloads the configuration without restart"], 
 ["reload <JAIL>", "reloads the jail <JAIL>"], 
 ["stop", "stops all jails and terminate the server"], 
 ["status", "gets the current status of the server"], 
-["ping", "tests if the server is alive"], 
+["ping", "tests if the server is alive"],
+["echo", "for internal usage, returns back and outputs a given string"],
 ["help", "return this output"], 
 ["version", "return the server version"],
 ['', "LOGGING", ""],
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 1d3e89d3..d7f212c8 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -67,6 +67,11 @@ class Server:
 			'FreeBSD': '/var/run/log',
 			'Linux': '/dev/log',
 		}
+		# todo: remove that, if test cases are fixed
+		self.setSyslogSocket("auto")
+		# Set logging level
+		self.setLogLevel("INFO")
+		self.setLogTarget("STDOUT")
 
 	def __sigTERMhandler(self, signum, frame):
 		logSys.debug("Caught signal %d. Exiting" % signum)
@@ -77,13 +82,27 @@ class Server:
 		self.flushLogs()
 
 	def start(self, sock, pidfile, force=False, conf={}):
-		# First set all logging parameters:
-		self.setSyslogSocket(conf.get("syslogsocket", "auto"))
-		self.setLogLevel(conf.get("loglevel", "INFO"))
-		self.setLogTarget(conf.get("logtarget", "STDOUT"))
+		# First set the mask to only allow access to owner
+		os.umask(0077)
+		# Second daemonize before logging etc, because it will close all handles:
+		if self.__daemon: # pragma: no cover
+			logSys.info("Starting in daemon mode")
+			ret = self.__createDaemon()
+			if not ret:
+				logSys.error("Could not create daemon")
+				raise ServerInitializationError("Could not create daemon")
+		
+		# Set all logging parameters (or use default if not specified):
+		self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket))
+		self.setLogLevel(conf.get("loglevel", self.__logLevel))
+		self.setLogTarget(conf.get("logtarget", self.__logTarget))
 
+		logSys.info("-"*50)
 		logSys.info("Starting Fail2ban v%s", version.version)
 		
+		if self.__daemon: # pragma: no cover
+			logSys.info("Daemon started")
+
 		# Install signal handlers
 		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
 		signal.signal(signal.SIGINT, self.__sigTERMhandler)
@@ -92,17 +111,6 @@ class Server:
 		# Ensure unhandled exceptions are logged
 		sys.excepthook = excepthook
 
-		# First set the mask to only allow access to owner
-		os.umask(0077)
-		if self.__daemon: # pragma: no cover
-			logSys.info("Starting in daemon mode")
-			ret = self.__createDaemon()
-			if ret:
-				logSys.info("Daemon started")
-			else:
-				logSys.error("Could not create daemon")
-				raise ServerInitializationError("Could not create daemon")
-		
 		# Creates a PID file.
 		try:
 			logSys.debug("Creating PID file %s" % pidfile)
@@ -139,11 +147,8 @@ class Server:
 		self.stopAllJail()
 
 		# Only now shutdown the logging.
-		try:
-			self.__loggingLock.acquire()
+		with self.__loggingLock:
 			logging.shutdown()
-		finally:
-			self.__loggingLock.release()
 
 	def addJail(self, name, backend):
 		self.__jails.add(name, backend, self.__db)
@@ -372,16 +377,15 @@ class Server:
 	# @param value the level
 	
 	def setLogLevel(self, value):
-		try:
-			self.__loggingLock.acquire()
-			getLogger("fail2ban").setLevel(
-				getattr(logging, value.upper()))
-		except AttributeError:
-			raise ValueError("Invalid log level")
-		else:
-			self.__logLevel = value.upper()
-		finally:
-			self.__loggingLock.release()
+		value = value.upper()
+		with self.__loggingLock:
+			if self.__logLevel == value:
+				return
+			try:
+				getLogger("fail2ban").setLevel(getattr(logging, value))
+				self.__logLevel = value
+			except AttributeError:
+				raise ValueError("Invalid log level")
 	
 	##
 	# Get the logging level.
@@ -390,11 +394,8 @@ class Server:
 	# @return the log level
 	
 	def getLogLevel(self):
-		try:
-			self.__loggingLock.acquire()
+		with self.__loggingLock:
 			return self.__logLevel
-		finally:
-			self.__loggingLock.release()
 
 	##
 	# Sets the logging target.
@@ -480,24 +481,21 @@ class Server:
 	# syslogsocket is the full path to the syslog socket
 	# @param syslogsocket the syslog socket path
 	def setSyslogSocket(self, syslogsocket):
-		self.__syslogSocket = syslogsocket
-		# Conditionally reload, logtarget depends on socket path when SYSLOG
-		return self.__logTarget != "SYSLOG"\
-			   or self.setLogTarget(self.__logTarget)
+		with self.__loggingLock:
+			if self.__syslogSocket == syslogsocket:
+				return True
+			self.__syslogSocket = syslogsocket
+			# Conditionally reload, logtarget depends on socket path when SYSLOG
+			return self.__logTarget != "SYSLOG"\
+				   or self.setLogTarget(self.__logTarget)
 
 	def getLogTarget(self):
-		try:
-			self.__loggingLock.acquire()
+		with self.__loggingLock:
 			return self.__logTarget
-		finally:
-			self.__loggingLock.release()
 
 	def getSyslogSocket(self):
-		try:
-			self.__loggingLock.acquire()
+		with self.__loggingLock:
 			return self.__syslogSocket
-		finally:
-			self.__loggingLock.release()
 
 	def flushLogs(self):
 		if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']:
@@ -552,7 +550,7 @@ class Server:
 		# child process, and this makes sure that it is effect even if the parent
 		# terminates quickly.
 		signal.signal(signal.SIGHUP, signal.SIG_IGN)
-		
+
 		try:
 			# Fork a child process so the parent can exit.  This will return control
 			# to the command line or shell.  This is required so that the new process
@@ -593,7 +591,7 @@ class Server:
 				os._exit(0)	  # Exit parent (the first child) of the second child.
 		else:
 			os._exit(0)		 # Exit parent of the first child.
-		
+	
 		# Close all open files.  Try the system configuration variable, SC_OPEN_MAX,
 		# for the maximum number of open files to close.  If it doesn't exist, use
 		# the default value (configurable).
@@ -625,3 +623,6 @@ class Server:
 
 class ServerInitializationError(Exception):
 	pass
+
+class ServerDaemonize(Exception):
+	pass
\ No newline at end of file
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 29d6d189..2d6f0d77 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -93,6 +93,8 @@ class Transmitter:
 				name = command[1]
 				self.__server.stopJail(name)
 			return None
+		elif command[0] == "echo":
+			return command[1:]
 		elif command[0] == "sleep":
 			value = command[1]
 			time.sleep(float(value))

From afa1cdc3ae658e8595e7ca788cda1566c65bbadb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Feb 2016 17:57:23 +0100
Subject: [PATCH 075/623] client/server (bin) test cases introduced, ultimate
 closes #1121, closes #1139 small code review and fixing of some bugs during
 client-server communication process (in the test cases);

---
 bin/fail2ban-client                      |   4 +-
 bin/fail2ban-server                      |   4 +-
 fail2ban/client/fail2banclient.py        | 146 ++++----
 fail2ban/client/fail2bancmdline.py       | 194 ++++++-----
 fail2ban/client/fail2banserver.py        |  41 ++-
 fail2ban/protocol.py                     |  29 +-
 fail2ban/server/server.py                |  46 ++-
 fail2ban/tests/fail2banclienttestcase.py | 411 +++++++++++++++++++++++
 fail2ban/tests/fail2banregextestcase.py  |  12 -
 fail2ban/tests/utils.py                  |  25 ++
 10 files changed, 707 insertions(+), 205 deletions(-)
 create mode 100644 fail2ban/tests/fail2banclienttestcase.py

diff --git a/bin/fail2ban-client b/bin/fail2ban-client
index 19e76a98..f5ae7946 100755
--- a/bin/fail2ban-client
+++ b/bin/fail2ban-client
@@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers"
 __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
 __license__ = "GPL"
 
-from fail2ban.client.fail2banclient import exec_command_line
+from fail2ban.client.fail2banclient import exec_command_line, sys
 
 if __name__ == "__main__":
-	exec_command_line()
+	exec_command_line(sys.argv)
diff --git a/bin/fail2ban-server b/bin/fail2ban-server
index 8e64d865..ffafabe2 100755
--- a/bin/fail2ban-server
+++ b/bin/fail2ban-server
@@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers"
 __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester"
 __license__ = "GPL"
 
-from fail2ban.client.fail2banserver import exec_command_line
+from fail2ban.client.fail2banserver import exec_command_line, sys
 
 if __name__ == "__main__":
-	exec_command_line()
+	exec_command_line(sys.argv)
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 7adbab95..736f8fd2 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -28,12 +28,20 @@ import socket
 import sys
 import time
 
+import threading
 from threading import Thread
 
 from ..version import version
 from .csocket import CSocket
 from .beautifier import Beautifier
-from .fail2bancmdline import Fail2banCmdLine, logSys, exit
+from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output
+
+MAX_WAITTIME = 30
+
+
+def _thread_name():
+	return threading.current_thread().__class__.__name__
+
 
 ##
 #
@@ -51,13 +59,13 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		self._beautifier = None
 
 	def dispInteractive(self):
-		print "Fail2Ban v" + version + " reads log file that contains password failure report"
-		print "and bans the corresponding IP addresses using firewall rules."
-		print
+		output("Fail2Ban v" + version + " reads log file that contains password failure report")
+		output("and bans the corresponding IP addresses using firewall rules.")
+		output("")
 
 	def __sigTERMhandler(self, signum, frame):
 		# Print a new line because we probably come from wait
-		print
+		output("")
 		logSys.warning("Caught signal %d. Exiting" % signum)
 		exit(-1)
 
@@ -85,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 					if ret[0] == 0:
 						logSys.debug("OK : " + `ret[1]`)
 						if showRet or c[0] == 'echo':
-							print beautifier.beautify(ret[1])
+							output(beautifier.beautify(ret[1]))
 					else:
 						logSys.error("NOK: " + `ret[1].args`)
 						if showRet:
-							print beautifier.beautifyError(ret[1])
+							output(beautifier.beautifyError(ret[1]))
 						streamRet = False
 				except socket.error:
 					if showRet or self._conf["verbose"] > 1:
@@ -182,10 +190,13 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				# Start server direct here in main thread (not fork):
 				self._server = Fail2banServer.startServerDirect(self._conf, False)
 
+		except ExitException:
+			pass
 		except Exception as e:
-			print
-			logSys.error("Exception while starting server foreground")
+			output("")
+			logSys.error("Exception while starting server " + ("background" if background else "foreground"))
 			logSys.error(e)
+			return False
 		finally:
 			self._alive = False
 
@@ -229,18 +240,18 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		elif len(cmd) == 1 and cmd[0] == "restart":
 
 			if self._conf.get("interactive", False):
-				print('  ## stop ... ')
+				output('  ## stop ... ')
 			self.__processCommand(['stop'])
 			self.__waitOnServer(False)
 			# in interactive mode reset config, to make full-reload if there something changed:
 			if self._conf.get("interactive", False):
-				print('  ## load configuration ... ')
+				output('  ## load configuration ... ')
 				self.resetConf()
 				ret = self.initCmdLine(self._argv)
 				if ret is not None:
 					return ret
 			if self._conf.get("interactive", False):
-				print('  ## start ... ')
+				output('  ## start ... ')
 			return self.__processCommand(['start'])
 
 		elif len(cmd) >= 1 and cmd[0] == "reload":
@@ -283,7 +294,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			return False
 		return True
 
-	def __waitOnServer(self, alive=True, maxtime=30):
+	def __waitOnServer(self, alive=True, maxtime=None):
+		if maxtime is None:
+			maxtime = MAX_WAITTIME
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
 		with VisualWait(self._conf["verbose"]) as vis:
@@ -301,53 +314,59 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 
 	def start(self, argv):
 		# Install signal handlers
-		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
-		signal.signal(signal.SIGINT, self.__sigTERMhandler)
+		_prev_signals = {}
+		if _thread_name() == '_MainThread':
+			for s in (signal.SIGTERM, signal.SIGINT):
+				_prev_signals[s] = signal.getsignal(s)
+				signal.signal(s, self.__sigTERMhandler)
+		try:
+			# Command line options
+			if self._argv is None:
+				ret = self.initCmdLine(argv)
+				if ret is not None:
+					return ret
 
-		# Command line options
-		if self._argv is None:
-			ret = self.initCmdLine(argv)
-			if ret is not None:
-				return ret
+			# Commands
+			args = self._args
 
-		# Commands
-		args = self._args
-
-		# Interactive mode
-		if self._conf.get("interactive", False):
-			try:
-				import readline
-			except ImportError:
-				logSys.error("Readline not available")
-				return False
-			try:
-				ret = True
-				if len(args) > 0:
-					ret = self.__processCommand(args)
-				if ret:
-					readline.parse_and_bind("tab: complete")
-					self.dispInteractive()
-					while True:
-						cmd = raw_input(self.PROMPT)
-						if cmd == "exit" or cmd == "quit":
-							# Exit
-							return True
-						if cmd == "help":
-							self.dispUsage()
-						elif not cmd == "":
-							try:
-								self.__processCommand(shlex.split(cmd))
-							except Exception, e:
-								logSys.error(e)
-			except (EOFError, KeyboardInterrupt):
-				print
-				return True
-		# Single command mode
-		else:
-			if len(args) < 1:
-				self.dispUsage()
-				return False
-			return self.__processCommand(args)
+			# Interactive mode
+			if self._conf.get("interactive", False):
+				try:
+					import readline
+				except ImportError:
+					logSys.error("Readline not available")
+					return False
+				try:
+					ret = True
+					if len(args) > 0:
+						ret = self.__processCommand(args)
+					if ret:
+						readline.parse_and_bind("tab: complete")
+						self.dispInteractive()
+						while True:
+							cmd = raw_input(self.PROMPT)
+							if cmd == "exit" or cmd == "quit":
+								# Exit
+								return True
+							if cmd == "help":
+								self.dispUsage()
+							elif not cmd == "":
+								try:
+									self.__processCommand(shlex.split(cmd))
+								except Exception, e:
+									logSys.error(e)
+				except (EOFError, KeyboardInterrupt):
+					output("")
+					return True
+			# Single command mode
+			else:
+				if len(args) < 1:
+					self.dispUsage()
+					return False
+				return self.__processCommand(args)
+		finally:
+			for s, sh in _prev_signals.iteritems():
+				signal.signal(s, sh)
 
 
 class ServerExecutionException(Exception):
@@ -361,7 +380,8 @@ class ServerExecutionException(Exception):
 class _VisualWait:
 	pos = 0
 	delta = 1
-	maxpos = 10
+	def __init__(self, maxpos=10):
+		self.maxpos = maxpos
 	def __enter__(self):
 		return self
 	def __exit__(self, *args):
@@ -390,14 +410,14 @@ class _NotVisualWait:
 	def heartbeat(self):
 		pass
 
-def VisualWait(verbose):
-	return _VisualWait() if verbose > 1 else _NotVisualWait()
+def VisualWait(verbose, *args, **kwargs):
+	return _VisualWait(*args, **kwargs) if verbose > 1 else _NotVisualWait()
 
 
-def exec_command_line(): # pragma: no cover - can't test main
+def exec_command_line(argv):
 	client = Fail2banClient()
 	# Exit with correct return value
-	if client.start(sys.argv):
+	if client.start(argv):
 		exit(0)
 	else:
 		exit(-1)
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index e23f9b19..2ed6a499 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -33,7 +33,12 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
+def output(s):
+	print(s)
+
 CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",)
+# Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc)
+PRODUCTION = True
 
 
 class Fail2banCmdLine():
@@ -71,50 +76,50 @@ class Fail2banCmdLine():
 			self.__dict__[o] = obj.__dict__[o]
 
 	def dispVersion(self):
-		print "Fail2Ban v" + version
-		print
-		print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors"
-		print "Copyright of modifications held by their respective authors."
-		print "Licensed under the GNU General Public License v2 (GPL)."
-		print
-		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
-		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
+		output("Fail2Ban v" + version)
+		output("")
+		output("Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors")
+		output("Copyright of modifications held by their respective authors.")
+		output("Licensed under the GNU General Public License v2 (GPL).")
+		output("")
+		output("Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>.")
+		output("Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>.")
 
 	def dispUsage(self):
 		""" Prints Fail2Ban command line options and exits
 		"""
 		caller = os.path.basename(self._argv[0])
-		print "Usage: "+caller+" [OPTIONS]" + (" <COMMAND>" if not caller.endswith('server') else "")
-		print
-		print "Fail2Ban v" + version + " reads log file that contains password failure report"
-		print "and bans the corresponding IP addresses using firewall rules."
-		print
-		print "Options:"
-		print "    -c <DIR>                configuration directory"
-		print "    -s <FILE>               socket path"
-		print "    -p <FILE>               pidfile path"
-		print "    --loglevel <LEVEL>      logging level"
-		print "    --logtarget <FILE>|STDOUT|STDERR|SYSLOG"
-		print "    --syslogsocket auto|<FILE>"
-		print "    -d                      dump configuration. For debugging"
-		print "    -i                      interactive mode"
-		print "    -v                      increase verbosity"
-		print "    -q                      decrease verbosity"
-		print "    -x                      force execution of the server (remove socket file)"
-		print "    -b                      start server in background (default)"
-		print "    -f                      start server in foreground"
-		print "    --async                 start server in async mode (for internal usage only, don't read configuration)"
-		print "    -h, --help              display this help message"
-		print "    -V, --version           print the version"
+		output("Usage: "+caller+" [OPTIONS]" + (" <COMMAND>" if not caller.endswith('server') else ""))
+		output("")
+		output("Fail2Ban v" + version + " reads log file that contains password failure report")
+		output("and bans the corresponding IP addresses using firewall rules.")
+		output("")
+		output("Options:")
+		output("    -c <DIR>                configuration directory")
+		output("    -s <FILE>               socket path")
+		output("    -p <FILE>               pidfile path")
+		output("    --loglevel <LEVEL>      logging level")
+		output("    --logtarget <FILE>|STDOUT|STDERR|SYSLOG")
+		output("    --syslogsocket auto|<FILE>")
+		output("    -d                      dump configuration. For debugging")
+		output("    -i                      interactive mode")
+		output("    -v                      increase verbosity")
+		output("    -q                      decrease verbosity")
+		output("    -x                      force execution of the server (remove socket file)")
+		output("    -b                      start server in background (default)")
+		output("    -f                      start server in foreground")
+		output("    --async                 start server in async mode (for internal usage only, don't read configuration)")
+		output("    -h, --help              display this help message")
+		output("    -V, --version           print the version")
 		
 		if not caller.endswith('server'):
-			print
-			print "Command:"
+			output("")
+			output("Command:")
 			# Prints the protocol
 			printFormatted()
 
-		print
-		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
+		output("")
+		output("Report bugs to https://github.com/fail2ban/fail2ban/issues")
 
 	def __getCmdLineOptions(self, optList):
 		""" Gets the command line options
@@ -147,71 +152,80 @@ class Fail2banCmdLine():
 				self._conf["background"] = False
 			elif o in ["-h", "--help"]:
 				self.dispUsage()
-				exit(0)
+				return True
 			elif o in ["-V", "--version"]:
 				self.dispVersion()
-				exit(0)
+				return True
+		return None
 
 	def initCmdLine(self, argv):
-		# First time?
-		initial = (self._argv is None)
-
-		# Command line options
-		self._argv = argv
-
-		# Reads the command line options.
 		try:
-			cmdOpts = 'hc:s:p:xfbdviqV'
-			cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version']
-			optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
-		except getopt.GetoptError:
-			self.dispUsage()
-			exit(-1)
+			# First time?
+			initial = (self._argv is None)
 
-		self.__getCmdLineOptions(optList)
+			# Command line options
+			self._argv = argv
+			logSys.info("Using start params %s", argv[1:])
 
-		if initial:
-			verbose = self._conf["verbose"]
-			if verbose <= 0:
-				logSys.setLevel(logging.ERROR)
-			elif verbose == 1:
-				logSys.setLevel(logging.WARNING)
-			elif verbose == 2:
-				logSys.setLevel(logging.INFO)
-			elif verbose == 3:
-				logSys.setLevel(logging.DEBUG)
-			else:
-				logSys.setLevel(logging.HEAVYDEBUG)
-			# Add the default logging handler to dump to stderr
-			logout = logging.StreamHandler(sys.stderr)
-			# set a format which is simpler for console use
-			formatter = logging.Formatter('%(levelname)-6s %(message)s')
-			# tell the handler to use this format
-			logout.setFormatter(formatter)
-			logSys.addHandler(logout)
+			# Reads the command line options.
+			try:
+				cmdOpts = 'hc:s:p:xfbdviqV'
+				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version']
+				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
+			except getopt.GetoptError:
+				self.dispUsage()
+				return False
 
-		# Set expected parameters (like socket, pidfile, etc) from configuration,
-		# if those not yet specified, in which read configuration only if needed here:
-		conf = None
-		for o in CONFIG_PARAMS:
-			if self._conf.get(o, None) is None:
-				if not conf:
-					self.configurator.readEarly()
-					conf = self.configurator.getEarlyOptions()
-				self._conf[o] = conf[o]
+			ret = self.__getCmdLineOptions(optList)
+			if ret is not None:
+				return ret
 
-		logSys.info("Using socket file %s", self._conf["socket"])
+			if initial and PRODUCTION: # pragma: no cover - can't test
+				verbose = self._conf["verbose"]
+				if verbose <= 0:
+					logSys.setLevel(logging.ERROR)
+				elif verbose == 1:
+					logSys.setLevel(logging.WARNING)
+				elif verbose == 2:
+					logSys.setLevel(logging.INFO)
+				elif verbose == 3:
+					logSys.setLevel(logging.DEBUG)
+				else:
+					logSys.setLevel(logging.HEAVYDEBUG)
+				# Add the default logging handler to dump to stderr
+				logout = logging.StreamHandler(sys.stderr)
+				# set a format which is simpler for console use
+				formatter = logging.Formatter('%(levelname)-6s %(message)s')
+				# tell the handler to use this format
+				logout.setFormatter(formatter)
+				logSys.addHandler(logout)
 
-		logSys.info("Using pid file %s, [%s] logging to %s",
-			self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"])
+			# Set expected parameters (like socket, pidfile, etc) from configuration,
+			# if those not yet specified, in which read configuration only if needed here:
+			conf = None
+			for o in CONFIG_PARAMS:
+				if self._conf.get(o, None) is None:
+					if not conf:
+						self.configurator.readEarly()
+						conf = self.configurator.getEarlyOptions()
+					self._conf[o] = conf[o]
 
-		if self._conf.get("dump", False):
-			ret, stream = self.readConfig()
-			self.dumpConfig(stream)
-			return ret
+			logSys.info("Using socket file %s", self._conf["socket"])
 
-		# Nothing to do here, process in client/server
-		return None
+			logSys.info("Using pid file %s, [%s] logging to %s",
+				self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"])
+
+			if self._conf.get("dump", False):
+				ret, stream = self.readConfig()
+				self.dumpConfig(stream)
+				return ret
+
+			# Nothing to do here, process in client/server
+			return None
+		except Exception as e:
+			output("ERROR: %s" % (e,))
+			#logSys.exception(e)
+			return False
 
 	def readConfig(self, jail=None):
 		# Read the configuration
@@ -244,4 +258,8 @@ class Fail2banCmdLine():
 			sys.exit(code)
 
 # global exit handler:
-exit = Fail2banCmdLine.exit
\ No newline at end of file
+exit = Fail2banCmdLine.exit
+
+
+class ExitException:
+	pass
\ No newline at end of file
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 6c1dd694..da8e57b8 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -29,7 +29,11 @@ from ..server.server import Server, ServerDaemonize
 from ..server.utils import Utils
 from .fail2bancmdline import Fail2banCmdLine, logSys, exit
 
+MAX_WAITTIME = 30
+
 SERVER = "fail2ban-server"
+
+
 ##
 # \mainpage Fail2Ban
 #
@@ -72,8 +76,15 @@ class Fail2banServer(Fail2banCmdLine):
 
 	@staticmethod
 	def startServerAsync(conf):
-		# Forks the current process.
-		pid = os.fork()
+		# Directory of client (to try the first start from the same directory as client):
+		startdir = sys.path[0]
+		if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory:
+			startdir = os.path.dirname(sys.argv[0])
+		# Forks the current process, don't fork if async specified (ex: test cases)
+		pid = 0
+		frk = not conf["async"]
+		if frk:
+			pid = os.fork()
 		if pid == 0:
 			args = list()
 			args.append(SERVER)
@@ -96,14 +107,20 @@ class Fail2banServer(Fail2banCmdLine):
 
 			try:
 				# Use the current directory.
-				exe = os.path.abspath(os.path.join(sys.path[0], SERVER))
+				exe = os.path.abspath(os.path.join(startdir, SERVER))
 				logSys.debug("Starting %r with args %r", exe, args)
-				os.execv(exe, args)
-			except OSError:
+				if frk:
+					os.execv(exe, args)
+				else:
+					os.spawnv(os.P_NOWAITO, exe, args)
+			except OSError as e:
 				try:
 					# Use the PATH env.
-					logSys.warning("Initial start attempt failed.  Starting %r with the same args", SERVER)
-					os.execvp(SERVER, args)
+					logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER)
+					if frk:
+						os.execvp(SERVER, args)
+					else:
+						os.spawnvp(os.P_NOWAITO, SERVER, args)
 				except OSError:
 					exit(-1)
 
@@ -143,8 +160,8 @@ class Fail2banServer(Fail2banCmdLine):
 				phase = dict()
 				logSys.debug('Configure via async client thread')
 				cli.configureServer(async=True, phase=phase)
-				# wait up to 30 seconds, do not continue if configuration is not 100% valid:
-				Utils.wait_for(lambda: phase.get('ready', None) is not None, 30)
+				# wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid:
+				Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME)
 				if not phase.get('start', False):
 					return False
 
@@ -158,7 +175,7 @@ class Fail2banServer(Fail2banCmdLine):
 
 			# wait for client answer "done":
 			if not async and cli:
-				Utils.wait_for(lambda: phase.get('done', None) is not None, 30)
+				Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME)
 				if not phase.get('done', False):
 					if server:
 						server.quit()
@@ -179,9 +196,9 @@ class Fail2banServer(Fail2banCmdLine):
 			logSys.error("Could not start %s", SERVER)
 		exit(code)
 
-def exec_command_line(): # pragma: no cover - can't test main
+def exec_command_line(argv):
 	server = Fail2banServer()
-	if server.start(sys.argv):
+	if server.start(argv):
 		exit(0)
 	else:
 		exit(-1)
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 857d5fa6..648666a1 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -26,6 +26,9 @@ __license__ = "GPL"
 
 import textwrap
 
+def output(s):
+	print(s)
+
 ##
 # Describes the protocol used to communicate with the server.
 
@@ -143,7 +146,7 @@ def printFormatted():
 	firstHeading = False
 	for m in protocol:
 		if m[0] == '' and firstHeading:
-			print
+			output("")
 		firstHeading = True
 		first = True
 		if len(m[0]) >= MARGIN:
@@ -154,7 +157,7 @@ def printFormatted():
 				first = False
 			else:
 				line = ' ' * (INDENT + MARGIN) + n.strip()
-			print line
+			output(line)
 
 
 ##
@@ -165,20 +168,20 @@ def printWiki():
 	for m in protocol:
 		if m[0] == '':
 			if firstHeading:
-				print "|}"
+				output("|}")
 			__printWikiHeader(m[1], m[2])
 			firstHeading = True
 		else:
-			print "|-"
-			print "| <span style=\"white-space:nowrap;\"><tt>" + m[0] + "</tt></span> || || " + m[1]
-	print "|}"
+			output("|-")
+			output("| <span style=\"white-space:nowrap;\"><tt>" + m[0] + "</tt></span> || || " + m[1])
+	output("|}")
 
 
 def __printWikiHeader(section, desc):
-	print
-	print "=== " + section + " ==="
-	print
-	print desc
-	print
-	print "{|"
-	print "| '''Command''' || || '''Description'''"
+	output("")
+	output("=== " + section + " ===")
+	output("")
+	output(desc)
+	output("")
+	output("{|")
+	output("| '''Command''' || || '''Description'''")
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index d7f212c8..d4786f29 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -24,6 +24,7 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
+import threading
 from threading import Lock, RLock
 import logging
 import logging.handlers
@@ -42,6 +43,10 @@ from ..helpers import getLogger, excepthook
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+DEF_SYSLOGSOCKET = "auto"
+DEF_LOGLEVEL = "INFO"
+DEF_LOGTARGET = "STDOUT"
+
 try:
 	from .database import Fail2BanDb
 except ImportError: # pragma: no cover
@@ -49,6 +54,10 @@ except ImportError: # pragma: no cover
 	Fail2BanDb = None
 
 
+def _thread_name():
+	return threading.current_thread().__class__.__name__
+
+
 class Server:
 	
 	def __init__(self, daemon = False):
@@ -67,11 +76,7 @@ class Server:
 			'FreeBSD': '/var/run/log',
 			'Linux': '/dev/log',
 		}
-		# todo: remove that, if test cases are fixed
-		self.setSyslogSocket("auto")
-		# Set logging level
-		self.setLogLevel("INFO")
-		self.setLogTarget("STDOUT")
+		self.__prev_signals = {}
 
 	def __sigTERMhandler(self, signum, frame):
 		logSys.debug("Caught signal %d. Exiting" % signum)
@@ -93,9 +98,12 @@ class Server:
 				raise ServerInitializationError("Could not create daemon")
 		
 		# Set all logging parameters (or use default if not specified):
-		self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket))
-		self.setLogLevel(conf.get("loglevel", self.__logLevel))
-		self.setLogTarget(conf.get("logtarget", self.__logTarget))
+		self.setSyslogSocket(conf.get("syslogsocket", 
+			self.__syslogSocket if self.__syslogSocket is not None else DEF_SYSLOGSOCKET))
+		self.setLogLevel(conf.get("loglevel", 
+			self.__logLevel if self.__logLevel is not None else DEF_LOGLEVEL))
+		self.setLogTarget(conf.get("logtarget", 
+			self.__logTarget if self.__logTarget is not None else DEF_LOGTARGET))
 
 		logSys.info("-"*50)
 		logSys.info("Starting Fail2ban v%s", version.version)
@@ -104,10 +112,10 @@ class Server:
 			logSys.info("Daemon started")
 
 		# Install signal handlers
-		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
-		signal.signal(signal.SIGINT, self.__sigTERMhandler)
-		signal.signal(signal.SIGUSR1, self.__sigUSR1handler)
-		
+		if _thread_name() == '_MainThread':
+			for s in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1):
+				self.__prev_signals[s] = signal.getsignal(s)
+				signal.signal(s, self.__sigTERMhandler if s != signal.SIGUSR1 else self.__sigUSR1handler)
 		# Ensure unhandled exceptions are logged
 		sys.excepthook = excepthook
 
@@ -150,6 +158,10 @@ class Server:
 		with self.__loggingLock:
 			logging.shutdown()
 
+		# Restore default signal handlers:
+		for s, sh in self.__prev_signals.iteritems():
+			signal.signal(s, sh)
+
 	def addJail(self, name, backend):
 		self.__jails.add(name, backend, self.__db)
 		if self.__db is not None:
@@ -405,8 +417,13 @@ class Server:
 	
 	def setLogTarget(self, target):
 		with self.__loggingLock:
+			# don't set new handlers if already the same
+			# or if "INHERITED" (foreground worker of the test cases, to prevent stop logging):
 			if self.__logTarget == target:
 				return True
+			if target == "INHERITED":
+				self.__logTarget = target
+				return True
 			# set a format which is simpler for console use
 			formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s")
 			if target == "SYSLOG":
@@ -549,7 +566,10 @@ class Server:
 		# We need to set this in the parent process, so it gets inherited by the
 		# child process, and this makes sure that it is effect even if the parent
 		# terminates quickly.
-		signal.signal(signal.SIGHUP, signal.SIG_IGN)
+		if _thread_name() == '_MainThread':
+			for s in (signal.SIGHUP,):
+				self.__prev_signals[s] = signal.getsignal(s)
+				signal.signal(s, signal.SIG_IGN)
 
 		try:
 			# Fork a child process so the parent can exit.  This will return control
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
new file mode 100644
index 00000000..7db48fe8
--- /dev/null
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -0,0 +1,411 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Fail2Ban developers
+
+__author__ = "Serg Brester"
+__copyright__ = "Copyright (c) 2014- Serg G. Brester (sebres), 2008- Fail2Ban Contributors"
+__license__ = "GPL"
+
+import fileinput
+import os
+import re
+import time
+import unittest
+
+from threading import Thread
+
+from ..client import fail2banclient, fail2banserver, fail2bancmdline
+from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait
+from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
+from .. import protocol
+from ..server import server
+from ..server.utils import Utils
+from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging
+
+
+STOCK_CONF_DIR = "config"
+STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf'))
+TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config")
+if STOCK:
+	CONF_DIR = STOCK_CONF_DIR
+else:
+	CONF_DIR = TEST_CONF_DIR
+
+CLIENT = "fail2ban-client"
+SERVER = "fail2ban-server"
+BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin")
+
+MAX_WAITTIME = 10
+MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME)
+
+##
+# Several wrappers and settings for proper testing:
+#
+
+fail2banclient.MAX_WAITTIME = \
+fail2banserver.MAX_WAITTIME = MAX_WAITTIME
+
+
+fail2bancmdline.logSys = \
+fail2banclient.logSys = \
+fail2banserver.logSys = logSys
+
+LOG_LEVEL = logSys.level
+
+server.DEF_LOGTARGET = "/dev/null"
+
+def _test_output(*args):
+	logSys.info(args[0])
+fail2bancmdline.output = \
+fail2banclient.output = \
+fail2banserver.output = \
+protocol.output = _test_output
+
+def _test_exit(code=0):
+	logSys.debug("Exit with code %s", code)
+	if code == 0:
+		raise ExitException()
+	else:
+		raise FailExitException()   
+fail2bancmdline.exit = \
+fail2banclient.exit = \
+fail2banserver.exit = _test_exit
+
+INTERACT = []
+def _test_raw_input(*args):
+	if len(INTERACT):
+		#print('--- interact command: ', INTERACT[0])
+		return INTERACT.pop(0)
+	else:
+		return "exit" 
+fail2banclient.raw_input = _test_raw_input
+
+# prevents change logging params, log capturing, etc:
+fail2bancmdline.PRODUCTION = False
+
+
+class ExitException(fail2bancmdline.ExitException):
+	pass
+class FailExitException(fail2bancmdline.ExitException):
+	pass
+
+
+def _out_file(fn): # pragma: no cover
+	logSys.debug('---- ' + fn + ' ----')
+	for line in fileinput.input(fn):
+		line = line.rstrip('\n')
+		logSys.debug(line)
+	logSys.debug('-'*30)
+
+def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
+	cfg = tmp+"/config"
+	if use_stock and STOCK:
+		# copy config:
+		def ig_dirs(dir, files):
+			return [f for f in files if not os.path.isfile(os.path.join(dir, f))]
+		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
+		os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d")
+		os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d")
+		# replace fail2ban params (database with memory):
+		r = re.compile(r'^dbfile\s*=')
+		for line in fileinput.input(cfg+"/fail2ban.conf", inplace=True):
+			line = line.rstrip('\n')
+			if r.match(line):
+				line = "dbfile = :memory:"
+			print(line)
+		# replace jail params (polling as backend to be fast in initialize):
+		r = re.compile(r'^backend\s*=')
+		for line in fileinput.input(cfg+"/jail.conf", inplace=True):
+			line = line.rstrip('\n')
+			if r.match(line):
+				line = "backend = polling"
+			print(line)
+	else:
+		# just empty config directory without anything (only fail2ban.conf/jail.conf):
+		os.mkdir(cfg)
+		f = open(cfg+"/fail2ban.conf", "wb")
+		f.write('\n'.join((
+			"[Definition]",
+			"loglevel = INFO",
+			"logtarget = " + logtarget,
+			"syslogsocket = auto",
+			"socket = "+tmp+"/f2b.sock",
+			"pidfile = "+tmp+"/f2b.pid",
+			"backend = polling",
+			"dbfile = :memory:",
+			"dbpurgeage = 1d",
+			"",
+		)))
+		f.close()
+		f = open(cfg+"/jail.conf", "wb")
+		f.write('\n'.join((
+			"[INCLUDES]", "",
+			"[DEFAULT]", "",
+			"",
+		)))
+		f.close()
+		if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG
+			_out_file(cfg+"/fail2ban.conf")
+			_out_file(cfg+"/jail.conf")
+	# parameters:
+	return ("-c", cfg, 
+					"--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
+					"-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid")
+
+
+class Fail2banClientTest(LogCaptureTestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		LogCaptureTestCase.setUp(self)
+
+	def tearDown(self):
+		"""Call after every test case."""
+		LogCaptureTestCase.tearDown(self)
+
+	def testClientUsage(self):
+		self.assertRaises(ExitException, _exec_client, 
+			(CLIENT, "-h",))
+		self.assertLogged("Usage: " + CLIENT)
+		self.assertLogged("Report bugs to ")
+
+	@withtmpdir
+	def testClientStartBackgroundInside(self, tmp):
+		# always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process)
+		# (we can't fork the test cases process):
+		startparams = _start_params(tmp, True)
+		# start:
+		self.assertRaises(ExitException, _exec_client, 
+			(CLIENT, "--async", "-b") + startparams + ("start",))
+		self.assertLogged("Server ready")
+		self.assertLogged("Exit with code 0")
+		try:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+		finally:
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
+
+	@withtmpdir
+	def testClientStartBackgroundCall(self, tmp):
+		global INTERACT
+		startparams = _start_params(tmp)
+		# start (without async in new process):
+		cmd = os.path.join(os.path.join(BIN), CLIENT)
+		logSys.debug('Start %s ...', cmd)
+		Utils.executeCmd((cmd,) + startparams + ("start",), 
+			timeout=MAX_WAITTIME, shell=False, output=False)
+		self.pruneLog()
+		try:
+			# echo from client (inside):
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertLogged("TEST-ECHO")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+			# interactive client chat with started server:
+			INTERACT += [
+				"echo INTERACT-ECHO",
+				"status",
+				"exit"
+			]
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("-i",))
+			self.assertLogged("INTERACT-ECHO")
+			self.assertLogged("Status", "Number of jail:")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+			# test reload and restart over interactive client:
+			INTERACT += [
+				"reload",
+				"restart",
+				"exit"
+			]
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("-i",))
+			self.assertLogged("Reading config files:")
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Server ready")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+		finally:
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
+
+	def _testClientStartForeground(self, tmp, startparams, phase):
+		# start and wait to end (foreground):
+		phase['start'] = True
+		self.assertRaises(ExitException, _exec_client, 
+			(CLIENT, "-f") + startparams + ("start",))
+		# end :
+		phase['end'] = True
+
+	@withtmpdir
+	def testClientStartForeground(self, tmp):
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
+		# because foreground block execution - start it in thread:
+		phase = dict()
+		Thread(name="_TestCaseWorker", 
+			target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start()
+		try:
+			# wait for start thread:
+			Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+			self.assertTrue(phase.get('start', None))
+			# wait for server (socket):
+			Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
+			self.assertLogged("Starting communication")
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("ping",))
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+		finally:
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			# wait for end:
+			Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+			self.assertTrue(phase.get('end', None))
+			self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+			self.assertLogged("Exit with code 0")
+
+	@withtmpdir
+	def testClientFailStart(self, tmp):
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "--async", "-c", tmp+"/miss", "start",))
+		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",))
+		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+
+	def testVisualWait(self):
+		sleeptime = 0.035
+		for verbose in (2, 0):
+			cntr = 15
+			with VisualWait(verbose, 5) as vis:
+				while cntr:
+					vis.heartbeat()
+					if verbose and not unittest.F2B.fast:
+						time.sleep(sleeptime)
+					cntr -= 1
+
+
+class Fail2banServerTest(LogCaptureTestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		LogCaptureTestCase.setUp(self)
+
+	def tearDown(self):
+		"""Call after every test case."""
+		LogCaptureTestCase.tearDown(self)
+
+	def testServerUsage(self):
+		self.assertRaises(ExitException, _exec_server, 
+			(SERVER, "-h",))
+		self.assertLogged("Usage: " + SERVER)
+		self.assertLogged("Report bugs to ")
+
+	@withtmpdir
+	def testServerStartBackground(self, tmp):
+		# don't add "--async" by start, because if will fork current process by daemonize
+		# (we can't fork the test cases process),
+		# because server started internal communication in new thread use INHERITED as logtarget here:
+		startparams = _start_params(tmp, logtarget="INHERITED")
+		# start:
+		self.assertRaises(ExitException, _exec_server, 
+			(SERVER, "-b") + startparams)
+		self.assertLogged("Server ready")
+		self.assertLogged("Exit with code 0")
+		try:
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
+		finally:
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
+
+	def _testServerStartForeground(self, tmp, startparams, phase):
+		# start and wait to end (foreground):
+		phase['start'] = True
+		self.assertRaises(ExitException, _exec_server, 
+			(SERVER, "-f") + startparams + ("start",))
+		# end :
+		phase['end'] = True
+	@withtmpdir
+	def testServerStartForeground(self, tmp):
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
+		# because foreground block execution - start it in thread:
+		phase = dict()
+		Thread(name="_TestCaseWorker", 
+			target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start()
+		try:
+			# wait for start thread:
+			Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+			self.assertTrue(phase.get('start', None))
+			# wait for server (socket):
+			Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
+			self.assertLogged("Starting communication")
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("ping",))
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+		finally:
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("stop",))
+			# wait for end:
+			Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+			self.assertTrue(phase.get('end', None))
+			self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+			self.assertLogged("Exit with code 0")
+
+	@withtmpdir
+	def testServerFailStart(self, tmp):
+		self.assertRaises(FailExitException, _exec_server, 
+			(SERVER, "-c", tmp+"/miss",))
+		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+
+		self.assertRaises(FailExitException, _exec_server, 
+			(SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",))
+		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 49d6a3a6..d9f4081f 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -23,19 +23,7 @@ __author__ = "Serg Brester"
 __copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors"
 __license__ = "GPL"
 
-from __builtin__ import open as fopen
-import unittest
-import getpass
 import os
-import sys
-import time
-import tempfile
-import uuid
-
-try:
-	from systemd import journal
-except ImportError:
-	journal = None
 
 from ..client import fail2banregex
 from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index a0036979..e373aa5f 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -26,10 +26,14 @@ import logging
 import optparse
 import os
 import re
+import tempfile
+import shutil
 import sys
 import time
 import unittest
+
 from StringIO import StringIO
+from functools import wraps
 
 from ..helpers import getLogger
 from ..server.ipdns import DNSUtils
@@ -71,6 +75,17 @@ class F2B(optparse.Values):
 		return wtime
 
 
+def withtmpdir(f):
+	@wraps(f)
+	def wrapper(self, *args, **kwargs):
+		tmp = tempfile.mkdtemp(prefix="f2b-temp")
+		try:
+			return f(self, tmp, *args, **kwargs)
+		finally:
+			# clean up
+			shutil.rmtree(tmp)
+	return wrapper
+
 def initTests(opts):
 	unittest.F2B = F2B(opts)
 	# --fast :
@@ -156,6 +171,7 @@ def gatherTests(regexps=None, opts=None):
 	from . import misctestcase
 	from . import databasetestcase
 	from . import samplestestcase
+	from . import fail2banclienttestcase
 	from . import fail2banregextestcase
 
 	if not regexps: # pragma: no cover
@@ -239,6 +255,9 @@ def gatherTests(regexps=None, opts=None):
 	# Filter Regex tests with sample logs
 	tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex))
 
+	# bin/fail2ban-client, bin/fail2ban-server
+	tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banClientTest))
+	tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banServerTest))
 	# bin/fail2ban-regex
 	tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest))
 
@@ -321,8 +340,11 @@ class LogCaptureTestCase(unittest.TestCase):
 		# Let's log everything into a string
 		self._log = StringIO()
 		logSys.handlers = [logging.StreamHandler(self._log)]
+		if self._old_level <= logging.DEBUG:
+			print("")
 		if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them!
 			logSys.handlers += self._old_handlers
+			logSys.debug('--'*40)
 		logSys.setLevel(getattr(logging, 'DEBUG'))
 
 	def tearDown(self):
@@ -386,6 +408,9 @@ class LogCaptureTestCase(unittest.TestCase):
 	def pruneLog(self):
 		self._log.truncate(0)
 
+	def pruneLog(self):
+		self._log.truncate(0)
+
 	def getLog(self):
 		return self._log.getvalue()
 

From 2fcb6358ff1958fd0029dd8c9280f4d87a1aa909 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Feb 2016 21:15:03 +0100
Subject: [PATCH 076/623] several bug fixed: fork in client-server test cases
 prohibited, all worker threads daemonized (to prevent hanging on exit).

---
 fail2ban/client/fail2banclient.py        |  27 +-
 fail2ban/client/fail2banserver.py        |  11 +-
 fail2ban/server/jailthread.py            |   2 +
 fail2ban/server/server.py                |   6 +-
 fail2ban/tests/action_d/test_smtp.py     |   1 +
 fail2ban/tests/fail2banclienttestcase.py | 373 ++++++++++++++---------
 fail2ban/tests/utils.py                  |   3 +-
 7 files changed, 255 insertions(+), 168 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 736f8fd2..4f4caf79 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -34,7 +34,7 @@ from threading import Thread
 from ..version import version
 from .csocket import CSocket
 from .beautifier import Beautifier
-from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output
+from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output
 
 MAX_WAITTIME = 30
 
@@ -108,8 +108,13 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						logSys.error(e)
 					return False
 		finally:
+			# prevent errors by close during shutdown (on exit command):
 			if client:
-				client.close()
+				try :
+					client.close()
+				except Exception as e:
+					if showRet or self._conf["verbose"] > 1:
+						logSys.debug(e)
 			if showRet or c[0] == 'echo':
 				sys.stdout.flush()
 		return streamRet
@@ -184,7 +189,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 					return False
 			else:
 				# In foreground mode we should make server/client communication in different threads:
-				Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start()
+				th = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False))
+				th.daemon = True
+				th.start()
 				# Mark current (main) thread as daemon:
 				self.setDaemon(True)
 				# Start server direct here in main thread (not fork):
@@ -197,8 +204,6 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			logSys.error("Exception while starting server " + ("background" if background else "foreground"))
 			logSys.error(e)
 			return False
-		finally:
-			self._alive = False
 
 		return True
 
@@ -206,7 +211,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 	def configureServer(self, async=True, phase=None):
 		# if asynchron start this operation in the new thread:
 		if async:
-			return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start()
+			th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase))
+			th.daemon = True
+			return th.start()
 		# prepare: read config, check configuration is valid, etc.:
 		if phase is not None:
 			phase['start'] = True
@@ -290,7 +297,6 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						 "server, adding the -x option will do it")
 			if self._server:
 				self._server.quit()
-				exit(-1)
 			return False
 		return True
 
@@ -299,10 +305,12 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			maxtime = MAX_WAITTIME
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
+		logSys.debug("__waitOnServer: %r", (alive, maxtime))
 		with VisualWait(self._conf["verbose"]) as vis:
-			while self._alive and not self.__ping() == alive or (
+			while self._alive and (
+				not self.__ping() == alive or (
 				not alive and os.path.exists(self._conf["socket"])
-			):
+			)):
 				now = time.time()
 				# Wonderful visual :)
 				if now > starttime + 1:
@@ -365,6 +373,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 					return False
 				return self.__processCommand(args)
 		finally:
+			self._alive = False
 			for s, sh in _prev_signals.iteritems():
 				signal.signal(s, sh)
 
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index da8e57b8..ac927251 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -25,15 +25,14 @@ import os
 import sys
 
 from ..version import version
-from ..server.server import Server, ServerDaemonize
+from ..server.server import Server
 from ..server.utils import Utils
-from .fail2bancmdline import Fail2banCmdLine, logSys, exit
+from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit
 
 MAX_WAITTIME = 30
 
 SERVER = "fail2ban-server"
 
-
 ##
 # \mainpage Fail2Ban
 #
@@ -51,6 +50,7 @@ class Fail2banServer(Fail2banCmdLine):
 
 	@staticmethod
 	def startServerDirect(conf, daemon=True):
+		logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon)
 		server = None
 		try:
 			# Start it in foreground (current thread, not new process),
@@ -59,8 +59,6 @@ class Fail2banServer(Fail2banCmdLine):
 			server.start(conf["socket"],
 							conf["pidfile"], conf["force"], 
 							conf=conf)
-		except ServerDaemonize:
-			pass
 		except Exception, e:
 			logSys.exception(e)
 			if server:
@@ -82,9 +80,10 @@ class Fail2banServer(Fail2banCmdLine):
 			startdir = os.path.dirname(sys.argv[0])
 		# Forks the current process, don't fork if async specified (ex: test cases)
 		pid = 0
-		frk = not conf["async"]
+		frk = not conf["async"] and PRODUCTION
 		if frk:
 			pid = os.fork()
+		logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid)
 		if pid == 0:
 			args = list()
 			args.append(SERVER)
diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py
index eb43e453..39a86c2b 100644
--- a/fail2ban/server/jailthread.py
+++ b/fail2ban/server/jailthread.py
@@ -51,6 +51,8 @@ class JailThread(Thread):
 
 	def __init__(self, name=None):
 		super(JailThread, self).__init__(name=name)
+		## Should going with main thread also:
+		self.daemon = True
 		## Control the state of the thread.
 		self.active = False
 		## Control the idle state of the thread.
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index d4786f29..9c970b49 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -351,6 +351,9 @@ class Server:
 	def getBanTime(self, name):
 		return self.__jails[name].actions.getBanTime()
 	
+	def isStarted(self):
+		self.__asyncServer.isActive()
+
 	def isAlive(self, jailnum=None):
 		if jailnum is not None and len(self.__jails) != jailnum:
 			return 0
@@ -643,6 +646,3 @@ class Server:
 
 class ServerInitializationError(Exception):
 	pass
-
-class ServerDaemonize(Exception):
-	pass
\ No newline at end of file
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 37fe0138..1385fe82 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -65,6 +65,7 @@ class SMTPActionTest(unittest.TestCase):
 		self._active = True
 		self._loop_thread = threading.Thread(
 			target=asyncserver.loop, kwargs={'active': lambda: self._active})
+		self._loop_thread.daemon = True
 		self._loop_thread.start()
 
 	def tearDown(self):
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 7db48fe8..fd8a074b 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -98,7 +98,9 @@ def _test_raw_input(*args):
 fail2banclient.raw_input = _test_raw_input
 
 # prevents change logging params, log capturing, etc:
-fail2bancmdline.PRODUCTION = False
+fail2bancmdline.PRODUCTION = \
+fail2banclient.PRODUCTION = \
+fail2banserver.PRODUCTION = False
 
 
 class ExitException(fail2bancmdline.ExitException):
@@ -117,9 +119,9 @@ def _out_file(fn): # pragma: no cover
 def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	cfg = tmp+"/config"
 	if use_stock and STOCK:
-		# copy config:
+		# copy config (sub-directories as alias):
 		def ig_dirs(dir, files):
-			return [f for f in files if not os.path.isfile(os.path.join(dir, f))]
+			return [f for f in files if os.path.isdir(os.path.join(dir, f))]
 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
 		os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d")
 		os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d")
@@ -169,6 +171,47 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 					"--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
 					"-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid")
 
+def _kill_srv(pidfile): # pragma: no cover
+	def _pid_exists(pid):
+		try:
+			os.kill(pid, 0)
+			return True
+		except OSError:
+			return False
+	logSys.debug("-- cleanup: %r", (pidfile, os.path.isdir(pidfile)))
+	if os.path.isdir(pidfile):
+		piddir = pidfile
+		pidfile = piddir + "/f2b.pid"
+		if not os.path.isfile(pidfile):
+			pidfile = piddir + "/fail2ban.pid"
+	if not os.path.isfile(pidfile):
+		logSys.debug("--- cleanup: no pidfile for %r", piddir)
+		return True
+	f = pid = None
+	try:
+		logSys.debug("--- cleanup pidfile: %r", pidfile)
+		f = open(pidfile)
+		pid = f.read().split()[1]
+		pid = int(pid)
+		logSys.debug("--- cleanup pid: %r", pid)
+		if pid <= 0:
+			raise ValueError('pid %s of %s is invalid' % (pid, pidfile))
+		if not _pid_exists(pid):
+			return True
+		## try to preper stop (have signal handler):
+		os.kill(pid, signal.SIGTERM)
+		## check still exists after small timeout:
+		if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3):
+			## try to kill hereafter:
+			os.kill(pid, signal.SIGKILL)
+		return not _pid_exists(pid)
+	except Exception as e:
+		sysLog.debug(e)
+	finally:
+		if f is not None:
+			f.close()
+	return True
+
 
 class Fail2banClientTest(LogCaptureTestCase):
 
@@ -188,126 +231,144 @@ class Fail2banClientTest(LogCaptureTestCase):
 
 	@withtmpdir
 	def testClientStartBackgroundInside(self, tmp):
-		# always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process)
-		# (we can't fork the test cases process):
-		startparams = _start_params(tmp, True)
-		# start:
-		self.assertRaises(ExitException, _exec_client, 
-			(CLIENT, "--async", "-b") + startparams + ("start",))
-		self.assertLogged("Server ready")
-		self.assertLogged("Exit with code 0")
 		try:
+			# always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process)
+			# (we can't fork the test cases process):
+			startparams = _start_params(tmp, True)
+			# start:
 			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
-		finally:
-			self.pruneLog()
-			# stop:
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("stop",))
-			self.assertLogged("Shutdown successful")
+				(CLIENT, "--async", "-b") + startparams + ("start",))
+			self.assertLogged("Server ready")
 			self.assertLogged("Exit with code 0")
+			try:
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+				self.assertRaises(FailExitException, _exec_client, 
+					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("stop",))
+				self.assertLogged("Shutdown successful")
+				self.assertLogged("Exit with code 0")
+		finally:
+			_kill_srv(tmp)
 
 	@withtmpdir
 	def testClientStartBackgroundCall(self, tmp):
-		global INTERACT
-		startparams = _start_params(tmp)
-		# start (without async in new process):
-		cmd = os.path.join(os.path.join(BIN), CLIENT)
-		logSys.debug('Start %s ...', cmd)
-		Utils.executeCmd((cmd,) + startparams + ("start",), 
-			timeout=MAX_WAITTIME, shell=False, output=False)
-		self.pruneLog()
 		try:
-			# echo from client (inside):
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertLogged("TEST-ECHO")
-			self.assertLogged("Exit with code 0")
-			self.pruneLog()
-			# interactive client chat with started server:
-			INTERACT += [
-				"echo INTERACT-ECHO",
-				"status",
-				"exit"
-			]
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("-i",))
-			self.assertLogged("INTERACT-ECHO")
-			self.assertLogged("Status", "Number of jail:")
-			self.assertLogged("Exit with code 0")
-			self.pruneLog()
-			# test reload and restart over interactive client:
-			INTERACT += [
-				"reload",
-				"restart",
-				"exit"
-			]
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("-i",))
-			self.assertLogged("Reading config files:")
-			self.assertLogged("Shutdown successful")
-			self.assertLogged("Server ready")
-			self.assertLogged("Exit with code 0")
+			global INTERACT
+			startparams = _start_params(tmp)
+			# start (without async in new process):
+			cmd = os.path.join(os.path.join(BIN), CLIENT)
+			logSys.debug('Start %s ...', cmd)
+			Utils.executeCmd((cmd,) + startparams + ("start",), 
+				timeout=MAX_WAITTIME, shell=False, output=False)
 			self.pruneLog()
+			try:
+				# echo from client (inside):
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+				self.assertLogged("TEST-ECHO")
+				self.assertLogged("Exit with code 0")
+				self.pruneLog()
+				# interactive client chat with started server:
+				INTERACT += [
+					"echo INTERACT-ECHO",
+					"status",
+					"exit"
+				]
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("-i",))
+				self.assertLogged("INTERACT-ECHO")
+				self.assertLogged("Status", "Number of jail:")
+				self.assertLogged("Exit with code 0")
+				self.pruneLog()
+				# test reload and restart over interactive client:
+				INTERACT += [
+					"reload",
+					"restart",
+					"exit"
+				]
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("-i",))
+				self.assertLogged("Reading config files:")
+				self.assertLogged("Shutdown successful")
+				self.assertLogged("Server ready")
+				self.assertLogged("Exit with code 0")
+				self.pruneLog()
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("stop",))
+				self.assertLogged("Shutdown successful")
+				self.assertLogged("Exit with code 0")
 		finally:
-			self.pruneLog()
-			# stop:
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("stop",))
-			self.assertLogged("Shutdown successful")
-			self.assertLogged("Exit with code 0")
+			_kill_srv(tmp)
 
 	def _testClientStartForeground(self, tmp, startparams, phase):
 		# start and wait to end (foreground):
+		logSys.debug("-- start of test worker")
 		phase['start'] = True
 		self.assertRaises(ExitException, _exec_client, 
 			(CLIENT, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
+		logSys.debug("-- end of test worker")
 
 	@withtmpdir
 	def testClientStartForeground(self, tmp):
-		# started directly here, so prevent overwrite test cases logger with "INHERITED"
-		startparams = _start_params(tmp, logtarget="INHERITED")
-		# because foreground block execution - start it in thread:
-		phase = dict()
-		Thread(name="_TestCaseWorker", 
-			target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start()
+		th = None
 		try:
-			# wait for start thread:
-			Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-			self.assertTrue(phase.get('start', None))
-			# wait for server (socket):
-			Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
-			self.assertLogged("Starting communication")
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("ping",))
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			# started directly here, so prevent overwrite test cases logger with "INHERITED"
+			startparams = _start_params(tmp, logtarget="INHERITED")
+			# because foreground block execution - start it in thread:
+			phase = dict()
+			th = Thread(name="_TestCaseWorker", 
+				target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase))
+			th.daemon = True
+			th.start()
+			try:
+				# wait for start thread:
+				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('start', None))
+				# wait for server (socket):
+				Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
+				self.assertLogged("Starting communication")
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("ping",))
+				self.assertRaises(FailExitException, _exec_client, 
+					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("stop",))
+				# wait for end:
+				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('end', None))
+				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
 		finally:
-			self.pruneLog()
-			# stop:
-			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT,) + startparams + ("stop",))
-			# wait for end:
-			Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-			self.assertTrue(phase.get('end', None))
-			self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-			self.assertLogged("Exit with code 0")
+			_kill_srv(tmp)
+			if th:
+				th.join()
 
 	@withtmpdir
 	def testClientFailStart(self, tmp):
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", tmp+"/miss", "start",))
-		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+		try:
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "--async", "-c", tmp+"/miss", "start",))
+			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
 
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",))
-		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",))
+			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+		finally:
+			_kill_srv(tmp)
 
 	def testVisualWait(self):
 		sleeptime = 0.035
@@ -339,73 +400,89 @@ class Fail2banServerTest(LogCaptureTestCase):
 
 	@withtmpdir
 	def testServerStartBackground(self, tmp):
-		# don't add "--async" by start, because if will fork current process by daemonize
-		# (we can't fork the test cases process),
-		# because server started internal communication in new thread use INHERITED as logtarget here:
-		startparams = _start_params(tmp, logtarget="INHERITED")
-		# start:
-		self.assertRaises(ExitException, _exec_server, 
-			(SERVER, "-b") + startparams)
-		self.assertLogged("Server ready")
-		self.assertLogged("Exit with code 0")
 		try:
+			# don't add "--async" by start, because if will fork current process by daemonize
+			# (we can't fork the test cases process),
+			# because server started internal communication in new thread use INHERITED as logtarget here:
+			startparams = _start_params(tmp, logtarget="INHERITED")
+			# start:
 			self.assertRaises(ExitException, _exec_server, 
-				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
-		finally:
-			self.pruneLog()
-			# stop:
-			self.assertRaises(ExitException, _exec_server, 
-				(SERVER,) + startparams + ("stop",))
-			self.assertLogged("Shutdown successful")
+				(SERVER, "-b") + startparams)
+			self.assertLogged("Server ready")
 			self.assertLogged("Exit with code 0")
+			try:
+				self.assertRaises(ExitException, _exec_server, 
+					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+				self.assertRaises(FailExitException, _exec_server, 
+					(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertRaises(ExitException, _exec_server, 
+					(SERVER,) + startparams + ("stop",))
+				self.assertLogged("Shutdown successful")
+				self.assertLogged("Exit with code 0")
+		finally:
+			_kill_srv(tmp)
 
 	def _testServerStartForeground(self, tmp, startparams, phase):
 		# start and wait to end (foreground):
+		logSys.debug("-- start of test worker")
 		phase['start'] = True
 		self.assertRaises(ExitException, _exec_server, 
 			(SERVER, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
+		logSys.debug("-- end of test worker")
+
 	@withtmpdir
 	def testServerStartForeground(self, tmp):
-		# started directly here, so prevent overwrite test cases logger with "INHERITED"
-		startparams = _start_params(tmp, logtarget="INHERITED")
-		# because foreground block execution - start it in thread:
-		phase = dict()
-		Thread(name="_TestCaseWorker", 
-			target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start()
+		th = None
 		try:
-			# wait for start thread:
-			Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-			self.assertTrue(phase.get('start', None))
-			# wait for server (socket):
-			Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
-			self.assertLogged("Starting communication")
-			self.assertRaises(ExitException, _exec_server, 
-				(SERVER,) + startparams + ("ping",))
-			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
-			self.assertRaises(ExitException, _exec_server, 
-				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+			# started directly here, so prevent overwrite test cases logger with "INHERITED"
+			startparams = _start_params(tmp, logtarget="INHERITED")
+			# because foreground block execution - start it in thread:
+			phase = dict()
+			th = Thread(name="_TestCaseWorker", 
+				target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase))
+			th.daemon = True
+			th.start()
+			try:
+				# wait for start thread:
+				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('start', None))
+				# wait for server (socket):
+				Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
+				self.assertLogged("Starting communication")
+				self.assertRaises(ExitException, _exec_server, 
+					(SERVER,) + startparams + ("ping",))
+				self.assertRaises(FailExitException, _exec_server, 
+					(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
+				self.assertRaises(ExitException, _exec_server, 
+					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertRaises(ExitException, _exec_server, 
+					(SERVER,) + startparams + ("stop",))
+				# wait for end:
+				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('end', None))
+				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
 		finally:
-			self.pruneLog()
-			# stop:
-			self.assertRaises(ExitException, _exec_server, 
-				(SERVER,) + startparams + ("stop",))
-			# wait for end:
-			Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-			self.assertTrue(phase.get('end', None))
-			self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-			self.assertLogged("Exit with code 0")
+			_kill_srv(tmp)
+			if th:
+				th.join()
 
 	@withtmpdir
 	def testServerFailStart(self, tmp):
-		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", tmp+"/miss",))
-		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+		try:
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER, "-c", tmp+"/miss",))
+			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
 
-		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",))
-		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",))
+			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+		finally:
+			_kill_srv(tmp)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index e373aa5f..61bd8abf 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -340,9 +340,8 @@ class LogCaptureTestCase(unittest.TestCase):
 		# Let's log everything into a string
 		self._log = StringIO()
 		logSys.handlers = [logging.StreamHandler(self._log)]
-		if self._old_level <= logging.DEBUG:
+		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
 			print("")
-		if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them!
 			logSys.handlers += self._old_handlers
 			logSys.debug('--'*40)
 		logSys.setLevel(getattr(logging, 'DEBUG'))

From 0b4143730de2ef252984e4a7bc1310db8331b618 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Feb 2016 21:51:32 +0100
Subject: [PATCH 077/623] some compatibility fixes (prevent forking of
 testcase-process, code review), wait 4 server ready, test cases fixed
 (py2/py3)

---
 fail2ban/client/fail2banclient.py        |  80 ++++++++++------
 fail2ban/client/fail2bancmdline.py       |   8 +-
 fail2ban/client/fail2banserver.py        | 108 +++++++++++++--------
 fail2ban/server/server.py                |  36 ++++---
 fail2ban/tests/fail2banclienttestcase.py | 116 +++++++++++++----------
 fail2ban/tests/utils.py                  |   6 +-
 6 files changed, 219 insertions(+), 135 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 4f4caf79..ad5cc57e 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -34,14 +34,18 @@ from threading import Thread
 from ..version import version
 from .csocket import CSocket
 from .beautifier import Beautifier
-from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output
+from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \
+	logSys, PRODUCTION, exit, output
 
 MAX_WAITTIME = 30
+PROMPT = "fail2ban> "
 
 
 def _thread_name():
 	return threading.current_thread().__class__.__name__
 
+def input_command():
+	return raw_input(PROMPT)
 
 ##
 #
@@ -49,8 +53,6 @@ def _thread_name():
 
 class Fail2banClient(Fail2banCmdLine, Thread):
 
-	PROMPT = "fail2ban> "
-
 	def __init__(self):
 		Fail2banCmdLine.__init__(self)
 		Thread.__init__(self)
@@ -91,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						client = CSocket(self._conf["socket"])
 					ret = client.send(c)
 					if ret[0] == 0:
-						logSys.debug("OK : " + `ret[1]`)
+						logSys.debug("OK : %r", ret[1])
 						if showRet or c[0] == 'echo':
 							output(beautifier.beautify(ret[1]))
 					else:
-						logSys.error("NOK: " + `ret[1].args`)
+						logSys.error("NOK: %r", ret[1].args)
 						if showRet:
 							output(beautifier.beautifyError(ret[1]))
 						streamRet = False
@@ -202,7 +204,10 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		except Exception as e:
 			output("")
 			logSys.error("Exception while starting server " + ("background" if background else "foreground"))
-			logSys.error(e)
+			if self._conf["verbose"] > 1:
+				logSys.exception(e)
+			else:
+				logSys.error(e)
 			return False
 
 		return True
@@ -249,7 +254,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			if self._conf.get("interactive", False):
 				output('  ## stop ... ')
 			self.__processCommand(['stop'])
-			self.__waitOnServer(False)
+			if not self.__waitOnServer(False):
+				logSys.error("Could not stop server")
+				return False
 			# in interactive mode reset config, to make full-reload if there something changed:
 			if self._conf.get("interactive", False):
 				output('  ## load configuration ... ')
@@ -286,10 +293,14 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 	def __processStartStreamAfterWait(self, *args):
 		try:
 			# Wait for the server to start
-			self.__waitOnServer()
+			if not self.__waitOnServer():
+				logSys.error("Could not find server, waiting failed")
+				return False
 				# Configure the server
 			self.__processCmd(*args)
-		except ServerExecutionException:
+		except ServerExecutionException as e:
+			if self._conf["verbose"] > 1:
+				logSys.exception(e)
 			logSys.error("Could not start server. Maybe an old "
 						 "socket file is still present. Try to "
 						 "remove " + self._conf["socket"] + ". If "
@@ -306,11 +317,13 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
 		logSys.debug("__waitOnServer: %r", (alive, maxtime))
+		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping() 
 		with VisualWait(self._conf["verbose"]) as vis:
-			while self._alive and (
-				not self.__ping() == alive or (
-				not alive and os.path.exists(self._conf["socket"])
-			)):
+			sltime = 0.0125 / 2
+			while self._alive:
+				runf = test()
+				if runf == alive:
+					return True
 				now = time.time()
 				# Wonderful visual :)
 				if now > starttime + 1:
@@ -318,7 +331,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				# f end time reached:
 				if now - starttime >= maxtime:
 					raise ServerExecutionException("Failed to start server")
-				time.sleep(0.1)
+				sltime = min(sltime * 2, 0.5)
+				time.sleep(sltime)
+		return False
 
 	def start(self, argv):
 		# Install signal handlers
@@ -332,27 +347,31 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			if self._argv is None:
 				ret = self.initCmdLine(argv)
 				if ret is not None:
-					return ret
+					if ret:
+						return True
+					raise ServerExecutionException("Init of command line failed")
 
 			# Commands
 			args = self._args
 
 			# Interactive mode
 			if self._conf.get("interactive", False):
-				try:
-					import readline
-				except ImportError:
-					logSys.error("Readline not available")
-					return False
+				# no readline in test:
+				if PRODUCTION: # pragma: no cover
+					try:
+						import readline
+					except ImportError:
+						raise ServerExecutionException("Readline not available")
 				try:
 					ret = True
 					if len(args) > 0:
 						ret = self.__processCommand(args)
 					if ret:
-						readline.parse_and_bind("tab: complete")
+						if PRODUCTION: # pragma: no cover
+							readline.parse_and_bind("tab: complete")
 						self.dispInteractive()
 						while True:
-							cmd = raw_input(self.PROMPT)
+							cmd = input_command()
 							if cmd == "exit" or cmd == "quit":
 								# Exit
 								return True
@@ -362,26 +381,31 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 								try:
 									self.__processCommand(shlex.split(cmd))
 								except Exception, e:
-									logSys.error(e)
+									if self._conf["verbose"] > 1:
+										logSys.exception(e)
+									else:
+										logSys.error(e)
 				except (EOFError, KeyboardInterrupt):
 					output("")
-					return True
+					raise
 			# Single command mode
 			else:
 				if len(args) < 1:
 					self.dispUsage()
 					return False
 				return self.__processCommand(args)
+		except Exception as e:
+			if self._conf["verbose"] > 1:
+				logSys.exception(e)
+			else:
+				logSys.error(e)
+			return False
 		finally:
 			self._alive = False
 			for s, sh in _prev_signals.iteritems():
 				signal.signal(s, sh)
 
 
-class ServerExecutionException(Exception):
-	pass
-
-
 ##
 # Wonderful visual :)
 #
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 2ed6a499..781d55fc 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -261,5 +261,9 @@ class Fail2banCmdLine():
 exit = Fail2banCmdLine.exit
 
 
-class ExitException:
-	pass
\ No newline at end of file
+class ExitException(Exception):
+	pass
+
+
+class ServerExecutionException(Exception):
+	pass
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index ac927251..73e528ca 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -24,10 +24,8 @@ __license__ = "GPL"
 import os
 import sys
 
-from ..version import version
-from ..server.server import Server
-from ..server.utils import Utils
-from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit
+from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \
+	logSys, PRODUCTION, exit
 
 MAX_WAITTIME = 30
 
@@ -44,13 +42,14 @@ class Fail2banServer(Fail2banCmdLine):
 	# 	Fail2banCmdLine.__init__(self)
 
 	##
-	# Start Fail2Ban server in main thread without fork (foreground).
+	# Start Fail2Ban server in main thread without fork (direct, it can fork itself in Server if daemon=True).
 	#
-	# Start the Fail2ban server in foreground (daemon mode or not).
+	# Start the Fail2ban server in background/foreground (daemon mode or not).
 
 	@staticmethod
 	def startServerDirect(conf, daemon=True):
 		logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon)
+		from ..server.server import Server
 		server = None
 		try:
 			# Start it in foreground (current thread, not new process),
@@ -59,11 +58,14 @@ class Fail2banServer(Fail2banCmdLine):
 			server.start(conf["socket"],
 							conf["pidfile"], conf["force"], 
 							conf=conf)
-		except Exception, e:
-			logSys.exception(e)
-			if server:
-				server.quit()
-			exit(-1)
+		except Exception as e:
+			try:
+				if server:
+					server.quit()
+			except Exception as e2:
+				if conf["verbose"] > 1:
+					logSys.exception(e2)
+			raise
 
 		return server
 
@@ -74,10 +76,6 @@ class Fail2banServer(Fail2banCmdLine):
 
 	@staticmethod
 	def startServerAsync(conf):
-		# Directory of client (to try the first start from the same directory as client):
-		startdir = sys.path[0]
-		if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory:
-			startdir = os.path.dirname(sys.argv[0])
 		# Forks the current process, don't fork if async specified (ex: test cases)
 		pid = 0
 		frk = not conf["async"] and PRODUCTION
@@ -103,25 +101,43 @@ class Fail2banServer(Fail2banCmdLine):
 			for o in ('loglevel', 'logtarget', 'syslogsocket'):
 				args.append("--"+o)
 				args.append(conf[o])
-
 			try:
-				# Use the current directory.
-				exe = os.path.abspath(os.path.join(startdir, SERVER))
+				# Directory of client (to try the first start from current or the same directory as client, and from relative bin):
+				exe = Fail2banServer.getServerPath()
+				if not frk:
+					# Wrapr args to use the same python version in client/server (important for multi-python systems):
+					args[0] = exe
+					exe = sys.executable
+					args[0:0] = [exe]
 				logSys.debug("Starting %r with args %r", exe, args)
 				if frk:
-					os.execv(exe, args)
+					return os.execv(exe, args)
 				else:
-					os.spawnv(os.P_NOWAITO, exe, args)
+					# use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork):
+					return os.spawnv(os.P_WAIT, exe, args)
 			except OSError as e:
-				try:
-					# Use the PATH env.
-					logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER)
-					if frk:
-						os.execvp(SERVER, args)
-					else:
-						os.spawnvp(os.P_NOWAITO, SERVER, args)
-				except OSError:
-					exit(-1)
+				# Use the PATH env.
+				logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER)
+				if frk:
+					return os.execvp(SERVER, args)
+				else:
+					del args[0]
+					args[0] = SERVER
+					return os.spawnvp(os.P_WAIT, SERVER, args)
+		return pid
+
+	@staticmethod
+	def getServerPath():
+		startdir = sys.path[0]
+		exe = os.path.abspath(os.path.join(startdir, SERVER))
+		if not os.path.isfile(exe): # may be uresolved in test-cases, so get relative starter (client):
+			startdir = os.path.dirname(sys.argv[0])
+			exe = os.path.abspath(os.path.join(startdir, SERVER))
+			if not os.path.isfile(exe): # may be uresolved in test-cases, so try to get relative bin-directory:
+				startdir = os.path.dirname(os.path.abspath(__file__))
+				startdir = os.path.join(os.path.dirname(os.path.dirname(startdir)), "bin")
+				exe = os.path.abspath(os.path.join(startdir, SERVER))
+		return exe
 
 	def _Fail2banClient(self):
 		from .fail2banclient import Fail2banClient
@@ -139,18 +155,24 @@ class Fail2banServer(Fail2banCmdLine):
 		args = self._args
 
 		cli = None
-		# If client mode - whole processing over client:
-		if len(args) or self._conf.get("interactive", False):
-			cli = self._Fail2banClient()
-			return cli.start(argv)
+		# Just start:
+		if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False):
+			pass
+		else:
+			# If client mode - whole processing over client:
+			if len(args) or self._conf.get("interactive", False):
+				cli = self._Fail2banClient()
+				return cli.start(argv)
 
 		# Start the server:
 		server = None
 		try:
-			# async = True, if started from client, should fork, daemonize, etc...
-			# background = True, if should start in new process, otherwise start in foreground
-			async = self._conf.get("async", False)
+			from ..server.utils import Utils
+			# background = True, if should be new process running in background, otherwise start in foreground
+			# process will be forked in daemonize, inside of Server module.
+			# async = True, if started from client, should...
 			background = self._conf["background"]
+			async = self._conf.get("async", False)
 			# If was started not from the client:
 			if not async:
 				# Start new thread with client to read configuration and
@@ -162,13 +184,14 @@ class Fail2banServer(Fail2banCmdLine):
 				# wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid:
 				Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME)
 				if not phase.get('start', False):
-					return False
+					raise ServerExecutionException('Async configuration of server failed')
 
 			# Start server, daemonize it, etc.
-			if async or not background:
-				server = Fail2banServer.startServerDirect(self._conf, background)
-			else:
-				Fail2banServer.startServerAsync(self._conf)
+			pid = os.getpid()
+			server = Fail2banServer.startServerDirect(self._conf, background)
+			# If forked - just exit other processes
+			if pid != os.getpid():
+				os._exit(0)
 			if cli:
 				cli._server = server
 
@@ -182,7 +205,8 @@ class Fail2banServer(Fail2banCmdLine):
 				logSys.debug('Starting server done')
 
 		except Exception, e:
-			logSys.exception(e)
+			if self._conf["verbose"] > 1:
+				logSys.exception(e)
 			if server:
 				server.quit()
 			exit(-1)
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 9c970b49..fa6ca43d 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -93,9 +93,15 @@ class Server:
 		if self.__daemon: # pragma: no cover
 			logSys.info("Starting in daemon mode")
 			ret = self.__createDaemon()
-			if not ret:
-				logSys.error("Could not create daemon")
-				raise ServerInitializationError("Could not create daemon")
+			# If forked parent - return here (parent process will configure server later):
+			if ret is None:
+				return False
+			# If error:
+			if not ret[0]:
+				err = "Could not create daemon %s", ret[1:]
+				logSys.error(err)
+				raise ServerInitializationError(err)
+			# We are daemon.
 		
 		# Set all logging parameters (or use default if not specified):
 		self.setSyslogSocket(conf.get("syslogsocket", 
@@ -159,8 +165,12 @@ class Server:
 			logging.shutdown()
 
 		# Restore default signal handlers:
-		for s, sh in self.__prev_signals.iteritems():
-			signal.signal(s, sh)
+		if _thread_name() == '_MainThread':
+			for s, sh in self.__prev_signals.iteritems():
+				signal.signal(s, sh)
+
+		# Prevent to call quit twice:
+		self.quit = lambda: False
 
 	def addJail(self, name, backend):
 		self.__jails.add(name, backend, self.__db)
@@ -569,10 +579,9 @@ class Server:
 		# We need to set this in the parent process, so it gets inherited by the
 		# child process, and this makes sure that it is effect even if the parent
 		# terminates quickly.
-		if _thread_name() == '_MainThread':
-			for s in (signal.SIGHUP,):
-				self.__prev_signals[s] = signal.getsignal(s)
-				signal.signal(s, signal.SIG_IGN)
+		for s in (signal.SIGHUP,):
+			self.__prev_signals[s] = signal.getsignal(s)
+			signal.signal(s, signal.SIG_IGN)
 
 		try:
 			# Fork a child process so the parent can exit.  This will return control
@@ -583,7 +592,7 @@ class Server:
 			# PGID.
 			pid = os.fork()
 		except OSError, e:
-			return((e.errno, e.strerror))	 # ERROR (return a tuple)
+			return (False, (e.errno, e.strerror))	 # ERROR (return a tuple)
 		
 		if pid == 0:	   # The first child.
 	
@@ -604,7 +613,7 @@ class Server:
 				# preventing the daemon from ever acquiring a controlling terminal.
 				pid = os.fork()		# Fork a second child.
 			except OSError, e:
-				return((e.errno, e.strerror))  # ERROR (return a tuple)
+				return (False, (e.errno, e.strerror))  # ERROR (return a tuple)
 		
 			if (pid == 0):	  # The second child.
 				# Ensure that the daemon doesn't keep any directory in use.  Failure
@@ -613,7 +622,8 @@ class Server:
 			else:
 				os._exit(0)	  # Exit parent (the first child) of the second child.
 		else:
-			os._exit(0)		 # Exit parent of the first child.
+			# Signal to exit, parent of the first child.
+			return None
 	
 		# Close all open files.  Try the system configuration variable, SC_OPEN_MAX,
 		# for the maximum number of open files to close.  If it doesn't exist, use
@@ -641,7 +651,7 @@ class Server:
 		os.open("/dev/null", os.O_RDONLY)	# standard input (0)
 		os.open("/dev/null", os.O_RDWR)		# standard output (1)
 		os.open("/dev/null", os.O_RDWR)		# standard error (2)
-		return True
+		return (True,)
 
 
 class ServerInitializationError(Exception):
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index fd8a074b..6fee732b 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -26,6 +26,7 @@ __license__ = "GPL"
 import fileinput
 import os
 import re
+import sys
 import time
 import unittest
 
@@ -50,10 +51,9 @@ else:
 
 CLIENT = "fail2ban-client"
 SERVER = "fail2ban-server"
-BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin")
+BIN = os.path.dirname(Fail2banServer.getServerPath())
 
-MAX_WAITTIME = 10
-MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME)
+MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
 
 ##
 # Several wrappers and settings for proper testing:
@@ -67,8 +67,6 @@ fail2bancmdline.logSys = \
 fail2banclient.logSys = \
 fail2banserver.logSys = logSys
 
-LOG_LEVEL = logSys.level
-
 server.DEF_LOGTARGET = "/dev/null"
 
 def _test_output(*args):
@@ -89,13 +87,13 @@ fail2banclient.exit = \
 fail2banserver.exit = _test_exit
 
 INTERACT = []
-def _test_raw_input(*args):
+def _test_input_command(*args):
 	if len(INTERACT):
-		#print('--- interact command: ', INTERACT[0])
+		#logSys.debug('--- interact command: %r', INTERACT[0])
 		return INTERACT.pop(0)
 	else:
 		return "exit" 
-fail2banclient.raw_input = _test_raw_input
+fail2banclient.input_command = _test_input_command
 
 # prevents change logging params, log capturing, etc:
 fail2bancmdline.PRODUCTION = \
@@ -142,7 +140,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	else:
 		# just empty config directory without anything (only fail2ban.conf/jail.conf):
 		os.mkdir(cfg)
-		f = open(cfg+"/fail2ban.conf", "wb")
+		f = open(cfg+"/fail2ban.conf", "w")
 		f.write('\n'.join((
 			"[Definition]",
 			"loglevel = INFO",
@@ -156,20 +154,20 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 			"",
 		)))
 		f.close()
-		f = open(cfg+"/jail.conf", "wb")
+		f = open(cfg+"/jail.conf", "w")
 		f.write('\n'.join((
 			"[INCLUDES]", "",
 			"[DEFAULT]", "",
 			"",
 		)))
 		f.close()
-		if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG
+		if logSys.level < logging.DEBUG: # if HEAVYDEBUG
 			_out_file(cfg+"/fail2ban.conf")
 			_out_file(cfg+"/jail.conf")
-	# parameters:
-	return ("-c", cfg, 
-					"--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
-					"-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid")
+	# parameters (sock/pid and config, increase verbosity, set log, etc.):
+	return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid",
+					"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
+	)
 
 def _kill_srv(pidfile): # pragma: no cover
 	def _pid_exists(pid):
@@ -206,14 +204,14 @@ def _kill_srv(pidfile): # pragma: no cover
 			os.kill(pid, signal.SIGKILL)
 		return not _pid_exists(pid)
 	except Exception as e:
-		sysLog.debug(e)
+		logSys.debug(e)
 	finally:
 		if f is not None:
 			f.close()
 	return True
 
 
-class Fail2banClientTest(LogCaptureTestCase):
+class Fail2banClientServerBase(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
@@ -223,6 +221,21 @@ class Fail2banClientTest(LogCaptureTestCase):
 		"""Call after every test case."""
 		LogCaptureTestCase.tearDown(self)
 
+	def _wait_for_srv(self, tmp, ready=True, startparams=None):
+		sock = tmp+"/f2b.sock"
+		# wait for server (socket):
+		ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME)
+		if not ret:
+			raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,))
+		if ready:
+			# wait for communication with worker ready:
+			ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME)
+			if not ret:
+				raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,))
+
+
+class Fail2banClientTest(Fail2banClientServerBase):
+
 	def testClientUsage(self):
 		self.assertRaises(ExitException, _exec_client, 
 			(CLIENT, "-h",))
@@ -232,12 +245,13 @@ class Fail2banClientTest(LogCaptureTestCase):
 	@withtmpdir
 	def testClientStartBackgroundInside(self, tmp):
 		try:
-			# always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process)
-			# (we can't fork the test cases process):
+			# use once the stock configuration (to test starting also)
 			startparams = _start_params(tmp, True)
 			# start:
 			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT, "--async", "-b") + startparams + ("start",))
+				(CLIENT, "-b") + startparams + ("start",))
+			# wait for server (socket and ready):
+			self._wait_for_srv(tmp, True, startparams=startparams)
 			self.assertLogged("Server ready")
 			self.assertLogged("Exit with code 0")
 			try:
@@ -245,6 +259,11 @@ class Fail2banClientTest(LogCaptureTestCase):
 					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
 				self.assertRaises(FailExitException, _exec_client, 
 					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+				self.pruneLog()
+				# start again (should fail):
+				self.assertRaises(FailExitException, _exec_client, 
+					(CLIENT, "-b") + startparams + ("start",))
+				self.assertLogged("Server already running")
 			finally:
 				self.pruneLog()
 				# stop:
@@ -260,11 +279,14 @@ class Fail2banClientTest(LogCaptureTestCase):
 		try:
 			global INTERACT
 			startparams = _start_params(tmp)
-			# start (without async in new process):
-			cmd = os.path.join(os.path.join(BIN), CLIENT)
+			# start (in new process, using the same python version):
+			cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
 			logSys.debug('Start %s ...', cmd)
-			Utils.executeCmd((cmd,) + startparams + ("start",), 
-				timeout=MAX_WAITTIME, shell=False, output=False)
+			cmd = cmd + startparams + ("start",)
+			Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			# wait for server (socket and ready):
+			self._wait_for_srv(tmp, True, startparams=cmd)
+			self.assertLogged("Server ready")
 			self.pruneLog()
 			try:
 				# echo from client (inside):
@@ -312,7 +334,7 @@ class Fail2banClientTest(LogCaptureTestCase):
 		# start and wait to end (foreground):
 		logSys.debug("-- start of test worker")
 		phase['start'] = True
-		self.assertRaises(ExitException, _exec_client, 
+		self.assertRaises(fail2bancmdline.ExitException, _exec_client, 
 			(CLIENT, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
@@ -334,9 +356,10 @@ class Fail2banClientTest(LogCaptureTestCase):
 				# wait for start thread:
 				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
 				self.assertTrue(phase.get('start', None))
-				# wait for server (socket):
-				Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
-				self.assertLogged("Starting communication")
+				# wait for server (socket and ready):
+				self._wait_for_srv(tmp, True, startparams=startparams)
+				self.pruneLog()
+				# several commands to server:
 				self.assertRaises(ExitException, _exec_client, 
 					(CLIENT,) + startparams + ("ping",))
 				self.assertRaises(FailExitException, _exec_client, 
@@ -382,15 +405,7 @@ class Fail2banClientTest(LogCaptureTestCase):
 					cntr -= 1
 
 
-class Fail2banServerTest(LogCaptureTestCase):
-
-	def setUp(self):
-		"""Call before every test case."""
-		LogCaptureTestCase.setUp(self)
-
-	def tearDown(self):
-		"""Call after every test case."""
-		LogCaptureTestCase.tearDown(self)
+class Fail2banServerTest(Fail2banClientServerBase):
 
 	def testServerUsage(self):
 		self.assertRaises(ExitException, _exec_server, 
@@ -401,15 +416,17 @@ class Fail2banServerTest(LogCaptureTestCase):
 	@withtmpdir
 	def testServerStartBackground(self, tmp):
 		try:
-			# don't add "--async" by start, because if will fork current process by daemonize
-			# (we can't fork the test cases process),
-			# because server started internal communication in new thread use INHERITED as logtarget here:
-			startparams = _start_params(tmp, logtarget="INHERITED")
-			# start:
-			self.assertRaises(ExitException, _exec_server, 
-				(SERVER, "-b") + startparams)
+			# to prevent fork of test-cases process, start server in background via command:
+			startparams = _start_params(tmp)
+			# start (in new process, using the same python version):
+			cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
+			logSys.debug('Start %s ...', cmd)
+			cmd = cmd + startparams + ("-b",)
+			Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			# wait for server (socket and ready):
+			self._wait_for_srv(tmp, True, startparams=cmd)
 			self.assertLogged("Server ready")
-			self.assertLogged("Exit with code 0")
+			self.pruneLog()
 			try:
 				self.assertRaises(ExitException, _exec_server, 
 					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
@@ -429,7 +446,7 @@ class Fail2banServerTest(LogCaptureTestCase):
 		# start and wait to end (foreground):
 		logSys.debug("-- start of test worker")
 		phase['start'] = True
-		self.assertRaises(ExitException, _exec_server, 
+		self.assertRaises(fail2bancmdline.ExitException, _exec_server, 
 			(SERVER, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
@@ -451,9 +468,10 @@ class Fail2banServerTest(LogCaptureTestCase):
 				# wait for start thread:
 				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
 				self.assertTrue(phase.get('start', None))
-				# wait for server (socket):
-				Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME)
-				self.assertLogged("Starting communication")
+				# wait for server (socket and ready):
+				self._wait_for_srv(tmp, True, startparams=startparams)
+				self.pruneLog()
+				# several commands to server:
 				self.assertRaises(ExitException, _exec_server, 
 					(SERVER,) + startparams + ("ping",))
 				self.assertRaises(FailExitException, _exec_server, 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 61bd8abf..2d096276 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -54,6 +54,10 @@ if not CONFIG_DIR:
 	else:
 		CONFIG_DIR = '/etc/fail2ban'
 
+# In not installed env (setup, test-cases) use fail2ban modules from main directory:
+if 1 or os.environ.get('PYTHONPATH', None) is None:
+	os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
+		os.path.abspath(__file__)))))
 
 class F2B(optparse.Values):
 	def __init__(self, opts={}):
@@ -343,7 +347,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
 			print("")
 			logSys.handlers += self._old_handlers
-			logSys.debug('--'*40)
+			logSys.debug('='*10 + ' %s ' + '='*20, self.id())
 		logSys.setLevel(getattr(logging, 'DEBUG'))
 
 	def tearDown(self):

From 22576d715089839da0720eedefcb4b49673ecf8f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Feb 2016 19:19:31 +0100
Subject: [PATCH 078/623] code review, timeout fix, better tracing (and test
 coverage) by start of server/client (with or without fork)

---
 fail2ban/client/fail2banclient.py        |  21 ++--
 fail2ban/client/fail2bancmdline.py       |  22 +++--
 fail2ban/client/fail2banserver.py        |  21 ++--
 fail2ban/tests/fail2banclienttestcase.py | 116 ++++++++++++++++++-----
 4 files changed, 129 insertions(+), 51 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index ad5cc57e..23d31dc5 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -37,14 +37,13 @@ from .beautifier import Beautifier
 from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \
 	logSys, PRODUCTION, exit, output
 
-MAX_WAITTIME = 30
 PROMPT = "fail2ban> "
 
 
 def _thread_name():
 	return threading.current_thread().__class__.__name__
 
-def input_command():
+def input_command(): # pragma: no cover
 	return raw_input(PROMPT)
 
 ##
@@ -101,13 +100,19 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						if showRet:
 							output(beautifier.beautifyError(ret[1]))
 						streamRet = False
-				except socket.error:
+				except socket.error as e:
 					if showRet or self._conf["verbose"] > 1:
-						self.__logSocketError()
+						if showRet or c != ["ping"]:
+							self.__logSocketError()
+						else:
+							logSys.debug(" -- ping failed -- %r", e)
 					return False
-				except Exception, e:
+				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
-						logSys.error(e)
+						if self._conf["verbose"] > 1:
+							logSys.exception(e)
+						else:
+							logSys.error(e)
 					return False
 		finally:
 			# prevent errors by close during shutdown (on exit command):
@@ -123,7 +128,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 
 	def __logSocketError(self):
 		try:
-			if os.access(self._conf["socket"], os.F_OK):
+			if os.access(self._conf["socket"], os.F_OK): # pragma: no cover
 				# This doesn't check if path is a socket,
 				#  but socket.error should be raised
 				if os.access(self._conf["socket"], os.W_OK):
@@ -313,7 +318,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 
 	def __waitOnServer(self, alive=True, maxtime=None):
 		if maxtime is None:
-			maxtime = MAX_WAITTIME
+			maxtime = self._conf["timeout"]
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
 		logSys.debug("__waitOnServer: %r", (alive, maxtime))
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 781d55fc..d49f0e3d 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -33,13 +33,14 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
-def output(s):
+def output(s): # pragma: no cover
 	print(s)
 
 CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",)
 # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc)
 PRODUCTION = True
 
+MAX_WAITTIME = 30
 
 class Fail2banCmdLine():
 
@@ -56,7 +57,8 @@ class Fail2banCmdLine():
 			"background": True,
 			"verbose": 1,
 			"socket": None,
-			"pidfile": None
+			"pidfile": None,
+			"timeout": MAX_WAITTIME
 		}
 
 	@property
@@ -109,6 +111,7 @@ class Fail2banCmdLine():
 		output("    -b                      start server in background (default)")
 		output("    -f                      start server in foreground")
 		output("    --async                 start server in async mode (for internal usage only, don't read configuration)")
+		output("    --timeout               timeout to wait for the server (for internal usage only, don't read configuration)")
 		output("    -h, --help              display this help message")
 		output("    -V, --version           print the version")
 		
@@ -126,8 +129,6 @@ class Fail2banCmdLine():
 		"""
 		for opt in optList:
 			o = opt[0]
-			if o == "--async":
-				self._conf["async"] = True
 			if o == "-c":
 				self._conf["conf"] = opt[1]
 			elif o == "-s":
@@ -150,6 +151,11 @@ class Fail2banCmdLine():
 				self._conf["background"] = True
 			elif o == "-f":
 				self._conf["background"] = False
+			elif o == "--async":
+				self._conf["async"] = True
+			elif o == "-timeout":
+				from ..mytime import MyTime
+				self._conf["timeout"] = MyTime.str2seconds(opt[1])
 			elif o in ["-h", "--help"]:
 				self.dispUsage()
 				return True
@@ -170,7 +176,7 @@ class Fail2banCmdLine():
 			# Reads the command line options.
 			try:
 				cmdOpts = 'hc:s:p:xfbdviqV'
-				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version']
+				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'timeout=', 'help', 'version']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
@@ -180,6 +186,8 @@ class Fail2banCmdLine():
 			if ret is not None:
 				return ret
 
+			logSys.debug("-- conf: %r, args: %r", self._conf, self._args)
+
 			if initial and PRODUCTION: # pragma: no cover - can't test
 				verbose = self._conf["verbose"]
 				if verbose <= 0:
@@ -246,11 +254,11 @@ class Fail2banCmdLine():
 	@staticmethod
 	def dumpConfig(cmd):
 		for c in cmd:
-			print c
+			output(c)
 		return True
 
 	@staticmethod
-	def exit(code=0):
+	def exit(code=0): # pragma: no cover - can't test
 		logSys.debug("Exit with code %s", code)
 		if os._exit:
 			os._exit(code)
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 73e528ca..a6c8a7c5 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -27,8 +27,6 @@ import sys
 from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \
 	logSys, PRODUCTION, exit
 
-MAX_WAITTIME = 30
-
 SERVER = "fail2ban-server"
 
 ##
@@ -114,16 +112,17 @@ class Fail2banServer(Fail2banCmdLine):
 					return os.execv(exe, args)
 				else:
 					# use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork):
-					return os.spawnv(os.P_WAIT, exe, args)
-			except OSError as e:
+					ret = os.spawnv(os.P_WAIT, exe, args)
+					if ret != 0:
+						raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe))
+					return 0
+			except OSError as e: # pragma: no cover
+				if not frk: #not PRODUCTION:
+					raise
 				# Use the PATH env.
 				logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER)
 				if frk:
 					return os.execvp(SERVER, args)
-				else:
-					del args[0]
-					args[0] = SERVER
-					return os.spawnvp(os.P_WAIT, SERVER, args)
 		return pid
 
 	@staticmethod
@@ -181,8 +180,8 @@ class Fail2banServer(Fail2banCmdLine):
 				phase = dict()
 				logSys.debug('Configure via async client thread')
 				cli.configureServer(async=True, phase=phase)
-				# wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid:
-				Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME)
+				# wait, do not continue if configuration is not 100% valid:
+				Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"])
 				if not phase.get('start', False):
 					raise ServerExecutionException('Async configuration of server failed')
 
@@ -197,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine):
 
 			# wait for client answer "done":
 			if not async and cli:
-				Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME)
+				Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"])
 				if not phase.get('done', False):
 					if server:
 						server.quit()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 6fee732b..aa507fe6 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -43,11 +43,6 @@ from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging
 
 STOCK_CONF_DIR = "config"
 STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf'))
-TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config")
-if STOCK:
-	CONF_DIR = STOCK_CONF_DIR
-else:
-	CONF_DIR = TEST_CONF_DIR
 
 CLIENT = "fail2ban-client"
 SERVER = "fail2ban-server"
@@ -59,9 +54,7 @@ MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
 # Several wrappers and settings for proper testing:
 #
 
-fail2banclient.MAX_WAITTIME = \
-fail2banserver.MAX_WAITTIME = MAX_WAITTIME
-
+fail2bancmdline.MAX_WAITTIME = MAX_WAITTIME-1
 
 fail2bancmdline.logSys = \
 fail2banclient.logSys = \
@@ -167,6 +160,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
 	return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid",
 					"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
+					"--timeout", str(fail2bancmdline.MAX_WAITTIME),
 	)
 
 def _kill_srv(pidfile): # pragma: no cover
@@ -199,7 +193,7 @@ def _kill_srv(pidfile): # pragma: no cover
 		## try to preper stop (have signal handler):
 		os.kill(pid, signal.SIGTERM)
 		## check still exists after small timeout:
-		if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3):
+		if not Utils.wait_for(lambda: not _pid_exists(pid), 1):
 			## try to kill hereafter:
 			os.kill(pid, signal.SIGKILL)
 		return not _pid_exists(pid)
@@ -222,25 +216,50 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		LogCaptureTestCase.tearDown(self)
 
 	def _wait_for_srv(self, tmp, ready=True, startparams=None):
-		sock = tmp+"/f2b.sock"
-		# wait for server (socket):
-		ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME)
-		if not ret:
-			raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,))
-		if ready:
-			# wait for communication with worker ready:
-			ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME)
+		try:
+			sock = tmp+"/f2b.sock"
+			# wait for server (socket):
+			ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME)
 			if not ret:
-				raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,))
+				raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,))
+			if ready:
+				# wait for communication with worker ready:
+				ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME)
+				if not ret:
+					raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,))
+		except: # pragma: no cover
+			log = tmp+"/f2b.log"
+			if os.path.isfile(log):
+				_out_file(log)
+			else:
+				logSys.debug("No log file %s to examine details of error", log)
+			raise
 
 
 class Fail2banClientTest(Fail2banClientServerBase):
 
+	def testConsistency(self):
+		self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), CLIENT)))
+		self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), SERVER)))
+
 	def testClientUsage(self):
 		self.assertRaises(ExitException, _exec_client, 
 			(CLIENT, "-h",))
 		self.assertLogged("Usage: " + CLIENT)
 		self.assertLogged("Report bugs to ")
+		self.pruneLog()
+		self.assertRaises(ExitException, _exec_client, 
+			(CLIENT, "-vq", "-V",))
+		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
+
+	@withtmpdir
+	def testClientDump(self, tmp):
+		# use here the stock configuration (if possible)
+		startparams = _start_params(tmp, True)
+		self.assertRaises(ExitException, _exec_client, 
+			((CLIENT,) + startparams + ("-vvd",)))
+		self.assertLogged("Loading files")
+		self.assertLogged("logtarget")
 
 	@withtmpdir
 	def testClientStartBackgroundInside(self, tmp):
@@ -271,6 +290,13 @@ class Fail2banClientTest(Fail2banClientServerBase):
 					(CLIENT,) + startparams + ("stop",))
 				self.assertLogged("Shutdown successful")
 				self.assertLogged("Exit with code 0")
+
+			self.pruneLog()
+			# stop again (should fail):
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			self.assertLogged("Failed to access socket path")
+			self.assertLogged("Is fail2ban running?")
 		finally:
 			_kill_srv(tmp)
 
@@ -278,12 +304,13 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	def testClientStartBackgroundCall(self, tmp):
 		try:
 			global INTERACT
-			startparams = _start_params(tmp)
+			startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
 			# start (in new process, using the same python version):
 			cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
 			logSys.debug('Start %s ...', cmd)
-			cmd = cmd + startparams + ("start",)
-			Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			cmd = cmd + startparams + ("--async", "start",)
+			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			self.assertTrue(len(ret) and ret[0])
 			# wait for server (socket and ready):
 			self._wait_for_srv(tmp, True, startparams=cmd)
 			self.assertLogged("Server ready")
@@ -383,13 +410,35 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	@withtmpdir
 	def testClientFailStart(self, tmp):
 		try:
+			# started directly here, so prevent overwrite test cases logger with "INHERITED"
+			startparams = _start_params(tmp, logtarget="INHERITED")
+
+			## wrong config directory
 			self.assertRaises(FailExitException, _exec_client, 
 				(CLIENT, "--async", "-c", tmp+"/miss", "start",))
 			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+			self.pruneLog()
 
+			## wrong socket
 			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",))
+				(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",))
 			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			self.pruneLog()
+
+			## already exists:
+			open(tmp+"/f2b.sock", 'a').close()
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",))
+			self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
+			self.pruneLog()
+			os.remove(tmp+"/f2b.sock")
+
+			## wrong option:
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "-s",))
+			self.assertLogged("Usage: ")
+			self.pruneLog()
+
 		finally:
 			_kill_srv(tmp)
 
@@ -417,12 +466,13 @@ class Fail2banServerTest(Fail2banClientServerBase):
 	def testServerStartBackground(self, tmp):
 		try:
 			# to prevent fork of test-cases process, start server in background via command:
-			startparams = _start_params(tmp)
+			startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
 			# start (in new process, using the same python version):
 			cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
 			logSys.debug('Start %s ...', cmd)
 			cmd = cmd + startparams + ("-b",)
-			Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			self.assertTrue(len(ret) and ret[0])
 			# wait for server (socket and ready):
 			self._wait_for_srv(tmp, True, startparams=cmd)
 			self.assertLogged("Server ready")
@@ -495,12 +545,28 @@ class Fail2banServerTest(Fail2banClientServerBase):
 	@withtmpdir
 	def testServerFailStart(self, tmp):
 		try:
+			# started directly here, so prevent overwrite test cases logger with "INHERITED"
+			startparams = _start_params(tmp, logtarget="INHERITED")
+
+			## wrong config directory
 			self.assertRaises(FailExitException, _exec_server, 
 				(SERVER, "-c", tmp+"/miss",))
 			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+			self.pruneLog()
 
+			## wrong socket
 			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",))
+				(SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",))
 			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			self.pruneLog()
+
+			## already exists:
+			open(tmp+"/f2b.sock", 'a').close()
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",))
+			self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
+			self.pruneLog()
+			os.remove(tmp+"/f2b.sock")
+
 		finally:
 			_kill_srv(tmp)

From 060ea085f4b63c04518ec446abfd75896bc673d1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Feb 2016 20:41:20 +0100
Subject: [PATCH 079/623] reader bug fix: prevent to silent "load" of not
 existing jail; coverage of test cases increased;

---
 fail2ban/client/configreader.py          |  4 +++-
 fail2ban/client/fail2banclient.py        | 20 ++++++++++----------
 fail2ban/client/fail2banserver.py        | 24 ++++++++++++------------
 fail2ban/client/jailreader.py            |  2 +-
 fail2ban/tests/fail2banclienttestcase.py | 21 +++++++++++++++++++++
 5 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 37e66249..bcad59c3 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -208,7 +208,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None):
+	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 		values = dict()
 		for optname in options:
 			if isinstance(options, (list,tuple)):
@@ -229,6 +229,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 					continue
 				values[optname] = v
 			except NoSectionError, e:
+				if shouldExist:
+					raise
 				# No "Definition" section or wrong basedir
 				logSys.error(e)
 				values[optname] = optvalue
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 23d31dc5..a8e0a331 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -64,7 +64,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		output("and bans the corresponding IP addresses using firewall rules.")
 		output("")
 
-	def __sigTERMhandler(self, signum, frame):
+	def __sigTERMhandler(self, signum, frame): # pragma: no cover
 		# Print a new line because we probably come from wait
 		output("")
 		logSys.warning("Caught signal %d. Exiting" % signum)
@@ -141,7 +141,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				logSys.error("Failed to access socket path: %s."
 							 " Is fail2ban running?",
 							 self._conf["socket"])
-		except Exception as e:
+		except Exception as e: # pragma: no cover
 			logSys.error("Exception while checking socket access: %s",
 						 self._conf["socket"])
 			logSys.error(e)
@@ -165,7 +165,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				"There is no directory %s to contain the socket file %s."
 				% (socket_dir, self._conf["socket"]))
 			return None
-		if not os.access(socket_dir, os.W_OK | os.X_OK):
+		if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover
 			logSys.error(
 				"Directory %s exists but not accessible for writing"
 				% (socket_dir,))
@@ -204,9 +204,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				# Start server direct here in main thread (not fork):
 				self._server = Fail2banServer.startServerDirect(self._conf, False)
 
-		except ExitException:
+		except ExitException: # pragma: no cover
 			pass
-		except Exception as e:
+		except Exception as e: # pragma: no cover
 			output("")
 			logSys.error("Exception while starting server " + ("background" if background else "foreground"))
 			if self._conf["verbose"] > 1:
@@ -259,7 +259,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			if self._conf.get("interactive", False):
 				output('  ## stop ... ')
 			self.__processCommand(['stop'])
-			if not self.__waitOnServer(False):
+			if not self.__waitOnServer(False): # pragma: no cover
 				logSys.error("Could not stop server")
 				return False
 			# in interactive mode reset config, to make full-reload if there something changed:
@@ -298,12 +298,12 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 	def __processStartStreamAfterWait(self, *args):
 		try:
 			# Wait for the server to start
-			if not self.__waitOnServer():
+			if not self.__waitOnServer(): # pragma: no cover
 				logSys.error("Could not find server, waiting failed")
 				return False
 				# Configure the server
 			self.__processCmd(*args)
-		except ServerExecutionException as e:
+		except ServerExecutionException as e: # pragma: no cover
 			if self._conf["verbose"] > 1:
 				logSys.exception(e)
 			logSys.error("Could not start server. Maybe an old "
@@ -385,12 +385,12 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 							elif not cmd == "":
 								try:
 									self.__processCommand(shlex.split(cmd))
-								except Exception, e:
+								except Exception, e: # pragma: no cover
 									if self._conf["verbose"] > 1:
 										logSys.exception(e)
 									else:
 										logSys.error(e)
-				except (EOFError, KeyboardInterrupt):
+				except (EOFError, KeyboardInterrupt): # pragma: no cover
 					output("")
 					raise
 			# Single command mode
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index a6c8a7c5..a511e017 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -56,7 +56,7 @@ class Fail2banServer(Fail2banCmdLine):
 			server.start(conf["socket"],
 							conf["pidfile"], conf["force"], 
 							conf=conf)
-		except Exception as e:
+		except Exception as e: # pragma: no cover
 			try:
 				if server:
 					server.quit()
@@ -77,7 +77,7 @@ class Fail2banServer(Fail2banCmdLine):
 		# Forks the current process, don't fork if async specified (ex: test cases)
 		pid = 0
 		frk = not conf["async"] and PRODUCTION
-		if frk:
+		if frk: # pragma: no cover
 			pid = os.fork()
 		logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid)
 		if pid == 0:
@@ -108,22 +108,20 @@ class Fail2banServer(Fail2banCmdLine):
 					exe = sys.executable
 					args[0:0] = [exe]
 				logSys.debug("Starting %r with args %r", exe, args)
-				if frk:
-					return os.execv(exe, args)
+				if frk: # pragma: no cover
+					os.execv(exe, args)
 				else:
 					# use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork):
 					ret = os.spawnv(os.P_WAIT, exe, args)
-					if ret != 0:
+					if ret != 0: # pragma: no cover
 						raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe))
-					return 0
 			except OSError as e: # pragma: no cover
 				if not frk: #not PRODUCTION:
 					raise
 				# Use the PATH env.
 				logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER)
-				if frk:
-					return os.execvp(SERVER, args)
-		return pid
+				if frk: # pragma: no cover
+					os.execvp(SERVER, args)
 
 	@staticmethod
 	def getServerPath():
@@ -189,7 +187,7 @@ class Fail2banServer(Fail2banCmdLine):
 			pid = os.getpid()
 			server = Fail2banServer.startServerDirect(self._conf, background)
 			# If forked - just exit other processes
-			if pid != os.getpid():
+			if pid != os.getpid(): # pragma: no cover
 				os._exit(0)
 			if cli:
 				cli._server = server
@@ -198,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine):
 			if not async and cli:
 				Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"])
 				if not phase.get('done', False):
-					if server:
+					if server: # pragma: no cover
 						server.quit()
 					exit(-1)
 				logSys.debug('Starting server done')
@@ -206,7 +204,9 @@ class Fail2banServer(Fail2banCmdLine):
 		except Exception, e:
 			if self._conf["verbose"] > 1:
 				logSys.exception(e)
-			if server:
+			else:
+				logSys.error(e)
+			if server: # pragma: no cover
 				server.quit()
 			exit(-1)
 
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index fda5d40c..f0cf293a 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -114,7 +114,7 @@ class JailReader(ConfigReader):
 		defsec["fail2ban_version"] = version
 
 		# Read first options only needed for merge defaults ('known/...' from filter):
-		self.__opts = ConfigReader.getOptions(self, self.__name, opts1st)
+		self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True)
 		if not self.__opts:
 			return False
 		
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index aa507fe6..82ffe9b3 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -347,6 +347,21 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				self.assertLogged("Server ready")
 				self.assertLogged("Exit with code 0")
 				self.pruneLog()
+				# test reload missing jail (interactive):
+				INTERACT += [
+					"reload ~~unknown~jail~fail~~",
+					"exit"
+				]
+				self.assertRaises(ExitException, _exec_client, 
+					(CLIENT,) + startparams + ("-i",))
+				self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
+				self.pruneLog()
+				# test reload missing jail (direct):
+				self.assertRaises(FailExitException, _exec_client, 
+					(CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~"))
+				self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
+				self.assertLogged("Exit with code -1")
+				self.pruneLog()
 			finally:
 				self.pruneLog()
 				# stop:
@@ -425,6 +440,12 @@ class Fail2banClientTest(Fail2banClientServerBase):
 			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
 			self.pruneLog()
 
+			## not running
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",))
+			self.assertLogged("Could not find server")
+			self.pruneLog()
+
 			## already exists:
 			open(tmp+"/f2b.sock", 'a').close()
 			self.assertRaises(FailExitException, _exec_client, 

From 53956501dae839638ee2359ece32190a9a2ba59c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Feb 2016 21:57:12 +0100
Subject: [PATCH 080/623] increase readability and details level by increased
 verbosity

---
 bin/fail2ban-testcases           | 12 +++++++----
 fail2ban/server/server.py        | 13 +++++++----
 fail2ban/tests/actiontestcase.py |  2 +-
 fail2ban/tests/servertestcase.py | 37 +++++++++++++++++++++-----------
 fail2ban/tests/utils.py          |  1 +
 5 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 606b0b06..3b18b7c2 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -119,10 +119,14 @@ else:
 
 # Custom log format for the verbose tests runs
 if verbosity > 1: # pragma: no cover
-	stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt))
-else: # pragma: no cover
-	# just prefix with the space
-	stdout.setFormatter(Formatter(fmt))
+	if verbosity > 3:
+		fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
+	if verbosity > 2:
+	  fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt
+	else:
+		fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
+#
+stdout.setFormatter(Formatter(fmt))
 logSys.addHandler(stdout)
 
 #
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index fa6ca43d..9377ad5b 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -67,7 +67,8 @@ class Server:
 		self.__db = None
 		self.__daemon = daemon
 		self.__transm = Transmitter(self)
-		self.__asyncServer = AsyncServer(self.__transm)
+		#self.__asyncServer = AsyncServer(self.__transm)
+		self.__asyncServer = None
 		self.__logLevel = None
 		self.__logTarget = None
 		self.__syslogSocket = None
@@ -137,6 +138,7 @@ class Server:
 		# Start the communication
 		logSys.debug("Starting communication")
 		try:
+			self.__asyncServer = AsyncServer(self.__transm)
 			self.__asyncServer.start(sock, force)
 		except AsyncServerException, e:
 			logSys.error("Could not start server: %s", e)
@@ -155,14 +157,17 @@ class Server:
 		# communications first (which should be ok anyways since we
 		# are exiting)
 		# See https://github.com/fail2ban/fail2ban/issues/7
-		self.__asyncServer.stop()
+		if self.__asyncServer is not None:
+			self.__asyncServer.stop()
+			self.__asyncServer = None
 
 		# Now stop all the jails
 		self.stopAllJail()
 
 		# Only now shutdown the logging.
-		with self.__loggingLock:
-			logging.shutdown()
+		if self.__logTarget is not None:
+			with self.__loggingLock:
+				logging.shutdown()
 
 		# Restore default signal handlers:
 		if _thread_name() == '_MainThread':
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index f3778022..cffabcf0 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -44,8 +44,8 @@ class CommandActionTest(LogCaptureTestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-		LogCaptureTestCase.tearDown(self)
 		self.__action.stop()
+		LogCaptureTestCase.tearDown(self)
 
 	def testSubstituteRecursiveTags(self):
 		aInfo = {
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 90f7aaf2..e253ae84 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -66,25 +66,18 @@ class TransmitterBase(unittest.TestCase):
 	
 	def setUp(self):
 		"""Call before every test case."""
+		#super(TransmitterBase, self).setUp()
 		self.transm = self.server._Server__transm
-		self.tmp_files = []
-		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter')
-		os.close(sock_fd)
-		self.tmp_files.append(sock_name)
-		pidfile_fd, pidfile_name = tempfile.mkstemp(
-			'fail2ban.pid', 'transmitter')
-		os.close(pidfile_fd)
-		self.tmp_files.append(pidfile_name)
-		self.server.start(sock_name, pidfile_name, force=False)
+		# To test thransmitter we don't need to start server...
+		#self.server.start('/dev/null', '/dev/null', force=False)
 		self.jailName = "TestJail1"
 		self.server.addJail(self.jailName, FAST_BACKEND)
 
 	def tearDown(self):
 		"""Call after every test case."""
+		# stop jails, etc.
 		self.server.quit()
-		for f in self.tmp_files:
-			if os.path.exists(f):
-				os.remove(f)
+		#super(TransmitterBase, self).tearDown()
 
 	def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False):
 		"""Process set/get commands and compare both return values 
@@ -796,10 +789,10 @@ class TransmitterLogging(TransmitterBase):
 
 	def setUp(self):
 		self.server = Server()
+		super(TransmitterLogging, self).setUp()
 		self.server.setLogTarget("/dev/null")
 		self.server.setLogLevel("CRITICAL")
 		self.server.setSyslogSocket("auto")
-		super(TransmitterLogging, self).setUp()
 
 	def testLogTarget(self):
 		logTargets = []
@@ -968,6 +961,24 @@ class LoggingTests(LogCaptureTestCase):
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
 
+	def testStartFailedSockExists(self):
+		tmp_files = []
+		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'f2b-test')
+		os.close(sock_fd)
+		tmp_files.append(sock_name)
+		pidfile_fd, pidfile_name = tempfile.mkstemp('fail2ban.pid', 'f2b-test')
+		os.close(pidfile_fd)
+		tmp_files.append(pidfile_name)
+		server = TestServer()
+		try:
+			server.start(sock_name, pidfile_name, force=False)
+			self.assertLogged("Server already running")
+		finally:
+			server.quit()
+			for f in tmp_files:
+				if os.path.exists(f):
+					os.remove(f)
+
 
 from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 2d096276..bfbef4de 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -353,6 +353,7 @@ class LogCaptureTestCase(unittest.TestCase):
 	def tearDown(self):
 		"""Call after every test case."""
 		# print "O: >>%s<<" % self._log.getvalue()
+		self.pruneLog()
 		logSys = getLogger("fail2ban")
 		logSys.handlers = self._old_handlers
 		logSys.level = self._old_level

From 0c2eeee8c7fc4b3a26083ef2f04ebde0c96560c6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 12:53:42 +0200
Subject: [PATCH 081/623] BF: fail2ban-client can't unserialize IPAddr objects
 - added IPAddr pickle-handler, that simple wrap IPAddr to the str

---
 fail2ban/server/ipdns.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index bca03428..880e3f30 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -233,6 +233,14 @@ class IPAddr(object):
 
 	def __str__(self):
 		return self.ntoa
+
+	def __reduce__(self):
+		"""IPAddr pickle-handler, that simple wrap IPAddr to the str
+
+		Returns a string as instance to be pickled, because fail2ban-client can't
+		unserialize IPAddr objects
+		"""
+		return (str, (self.ntoa,))
 	
 	@property
 	def addr(self):

From 4b5b16cd9fc5baa8b43415de072d7ba74377943e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 15:33:15 +0200
Subject: [PATCH 082/623] allow using of IPv6 address style mask (analog to the
 IPv4), for example: `2606:28ff::/ffff:ff80::` -> `2606:2880::/25` fast
 calculating of maskplen using map table MAP_ADDR2MASKPLEN, with
 pre-calculated addr->maskplen values; test cases extended;

---
 fail2ban/server/ipdns.py         | 28 ++++++++++++++++++++--------
 fail2ban/tests/filtertestcase.py | 13 +++++++++++++
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 880e3f30..8109c21a 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -171,7 +171,7 @@ class IPAddr(object):
 		# IP address without CIDR mask
 		if len(s) > 2:
 			raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,))
-		if "." in s[1]: # 255.255.255.0 style mask
+		if "." in s[1] or ":" in s[1]: # 255.255.255.0 resp. ffff:: style mask
 			s[1] = IPAddr.masktoplen(s[1])
 		s[1] = long(s[1])
 		return s
@@ -235,7 +235,7 @@ class IPAddr(object):
 		return self.ntoa
 
 	def __reduce__(self):
-		"""IPAddr pickle-handler, that simple wrap IPAddr to the str
+		"""IPAddr pickle-handler, that simply wraps IPAddr to the str
 
 		Returns a string as instance to be pickled, because fail2ban-client can't
 		unserialize IPAddr objects
@@ -392,17 +392,29 @@ class IPAddr(object):
 		
 		return (self.addr & mask) == net.addr
 
+	# Pre-calculated map: addr to maskplen
+	def __getMaskMap():
+		m6 = (1 << 128)-1
+		m4 = (1 << 32)-1
+		mmap = {m6: 128, m4: 32, 0: 0}
+		m = 0
+		for i in xrange(0, 128):
+			m |= 1 << i
+			if i < 32:
+				mmap[m ^ m4] = 32-1-i
+			mmap[m ^ m6] = 128-1-i
+		return mmap
+
+	MAP_ADDR2MASKPLEN = __getMaskMap()
+
 	@property
 	def maskplen(self):
 		mplen = 0
 		if self._maskplen is not None:
 			return self._maskplen
-		maddr = self._addr
-		while maddr:
-			if not (maddr & 0x80000000):
-				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
-			maddr = (maddr << 1) & 0xFFFFFFFFL
-			mplen += 1
+		mplen = IPAddr.MAP_ADDR2MASKPLEN.get(self._addr)
+		if mplen is None:
+			raise ValueError("invalid mask %r, no plen representation" % (str(self),))
 		self._maskplen = mplen
 		return mplen
 		
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 5e493f83..1d4db3a2 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1452,11 +1452,24 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(IPAddr('93.184.0.1', 24).ntoa, '93.184.0.0/24')
 		self.assertEqual(IPAddr('192.168.1.0/255.255.255.128').ntoa, '192.168.1.0/25')
 
+		self.assertEqual(IPAddr('93.184.0.1/32').ntoa, '93.184.0.1')
+		self.assertEqual(IPAddr('93.184.0.1/255.255.255.255').ntoa, '93.184.0.1')
+
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8::', 120)), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8::', 120).ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
 		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
 
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::', 25)), '2606:2880::/25')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ff80::')), '2606:2880::/25')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff::')), 
+			'2606:28ff:220:1:248:1893:25c8:0/112')
+
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/128')), 
+			'2606:28ff:220:1:248:1893:25c8:0')
+		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')), 
+			'2606:28ff:220:1:248:1893:25c8:0')
+
 	def testIPAddr_CIDR_Repr(self):
 		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
 			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]

From ec6032d934f72fc2a373a630e44499505b561519 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 17:16:09 +0200
Subject: [PATCH 083/623] prevent to fail stock configs test case, if any jail
 custom config does not have own test log-file (perhaps not clean copy)

---
 fail2ban/tests/servertestcase.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 90f7aaf2..b79a5652 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1021,7 +1021,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						cmd[2] = 'polling'
 					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
 					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
-						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
+						fn = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
+						# fallback to testcase01 if jail has not an own test log-file (currently should be no matter):
+						if not os.path.exists(fn): # pragma: no cover
+							fn = os.path.join(TEST_FILES_DIR, 'testcase01.log')
+						cmd[3] = fn
 					# if fast add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
 					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'

From 14c31d8c58ac1b5beb3974a8b3ecd82ab19d7a08 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 12 May 2016 18:52:20 -0400
Subject: [PATCH 084/623] DOC: minor PEP8ing and comments enhancements

---
 fail2ban/tests/clientreadertestcase.py |  2 +-
 fail2ban/tests/servertestcase.py       | 20 ++++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 0edbc69e..5e66e25e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -45,7 +45,7 @@ TEST_FILES_DIR_SHARE_CFG = {}
 from .utils import CONFIG_DIR
 CONFIG_DIR_SHARE_CFG = unittest.F2B.share_config
 
-STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
+STOCK = os.path.exists(os.path.join('config', 'fail2ban.conf'))
 
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
 IMPERFECT_CONFIG_SHARE_CFG = {}
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index b79a5652..214520a6 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -844,7 +844,7 @@ class TransmitterLogging(TransmitterBase):
 				   outCode=1,
 				   outValue=Exception('Failed to change log target'),
 				   repr_=True # Exceptions are not comparable apparently
-                                  )
+                                )
 			  }[platform.system() in ('Linux',) and os.path.exists('/dev/log')]
 		)
 
@@ -1001,7 +1001,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 	if STOCK:
 
 		def testCheckStockJailActions(self):
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			# we are running tests from root project dir atm
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg)
 			self.assertTrue(jails.read())		  # opens fine
 			self.assertTrue(jails.getOptions())	  # reads fine
 			stream = jails.convert(allow_no_files=True)
@@ -1019,14 +1020,17 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# change to the fast init backend:
 					if cmd[0] == 'add':
 						cmd[2] = 'polling'
-					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
+					# change log path to test log of the jail
+					# (to prevent "Permission denied" on /var/logs/ for test-user):
 					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
 						fn = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
-						# fallback to testcase01 if jail has not an own test log-file (currently should be no matter):
-						if not os.path.exists(fn): # pragma: no cover
+						# fallback to testcase01 if jail has no its own test log-file
+						# (should not matter really):
+						if not os.path.exists(fn):  # pragma: no cover
 							fn = os.path.join(TEST_FILES_DIR, 'testcase01.log')
 						cmd[3] = fn
-					# if fast add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					# if fast add dummy regex to prevent too long compile of all regexp
+					# (we don't use it in this test at all):
 					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
 					):
@@ -1035,7 +1039,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
 					try:
 						cmdHandler(cmd)
-					except Exception, e: # pragma: no cover
+					except Exception, e:  # pragma: no cover
 						self.fail("Command %r has failed. Received %r" % (cmd, e))
 
 			# jails = server._Server__jails
@@ -1318,7 +1322,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
- 						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',

From 3d3735706b71da567de00b0158beba8915da4c94 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 May 2016 20:21:42 +0200
Subject: [PATCH 085/623] invalid recursion check in substituteRecursiveTags:
 for example action `bsd-ipfw` produced ValueError('properties contain self
 referencing definitions and cannot be resolved...') test cases extended for
 exactly this case and for all stock actions; closes gh-1417

---
 fail2ban/server/action.py        | 14 +++++---
 fail2ban/tests/actiontestcase.py |  7 ++++
 fail2ban/tests/servertestcase.py | 57 +++++++++++++++++++++++++++++++-
 3 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index cdd4e28c..ba3d4633 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -366,17 +366,22 @@ class CommandAction(ActionBase):
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = []
+				done = {}
+				last_found = tag
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or found_tag in done:
+					curdone = done.get(last_found)
+					if curdone is None:
+						done[last_found] = curdone = []
+					if found_tag == tag or found_tag in curdone:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						raise ValueError(
 							"properties contain self referencing definitions "
-							"and cannot be resolved, fail tag: %s value: %s" % (tag, value))
+							"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
+							(tag, found_tag, curdone, value))
 					repl = None
 					if found_tag not in cls._escapedTags:
 						repl = tags.get(found_tag + '?' + conditional)
@@ -390,7 +395,8 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
-					done.append(found_tag)
+					curdone.append(found_tag)
+					last_found = found_tag
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index f3778022..00a4d6be 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -30,6 +30,7 @@ import time
 import unittest
 
 from ..server.action import CommandAction, CallingMap
+from ..server.actions import OrderedDict
 from ..server.utils import Utils
 
 from .utils import LogCaptureTestCase
@@ -65,6 +66,12 @@ class CommandActionTest(LogCaptureTestCase):
 			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertRaises(ValueError,
 			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		# No-recursion, just multiple replacement of tag <T>, should be successful
+		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
+			self.assertEqual(CommandAction.substituteRecursiveTags(
+				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			)
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 214520a6..cc14e97f 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -998,6 +998,38 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
 		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
 
+	def _testExecActions(self, server):
+		jails = server._Server__jails
+		for jail in jails:
+			# print(jail, jails[jail])
+			for a in jails[jail].actions:
+				action = jails[jail].actions[a]
+				logSys.debug('# ' + ('=' * 50))
+				logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+				logSys.debug('# ' + ('=' * 50))
+				# we can currently test only command actions:
+				if not isinstance(action, _actions.CommandAction): continue
+				# wrap default command processor, just log if (heavy)debug:
+				action.executeCmd = self._executeCmd
+				# test start :
+				logSys.debug('# === start ==='); self.pruneLog()
+				action.start()
+				# test ban ip4 :
+				logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+				action.ban({'ip': IPAddr('192.0.2.1')})
+				# test unban ip4 :
+				logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+				action.unban({'ip': IPAddr('192.0.2.1')})
+				# test ban ip6 :
+				logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+				action.ban({'ip': IPAddr('2001:DB8::')})
+				# test unban ip6 :
+				logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+				action.unban({'ip': IPAddr('2001:DB8::')})
+				# test stop :
+				logSys.debug('# === stop ==='); self.pruneLog()
+				action.stop()
+
 	if STOCK:
 
 		def testCheckStockJailActions(self):
@@ -1046,6 +1078,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			# for j in jails:
 			# 	print(j, jails[j])
 
+			# test default stock actions sepecified in all stock jails:
+			if not unittest.F2B.fast:
+				self._testExecActions(server)
+
 		def getDefaultJailStream(self, jail, act):
 			act = act.replace('%(__name__)s', jail)
 			actName, actOpt = JailReader.extractOptions(act)
@@ -1061,6 +1097,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			stream.extend(action.convert())
 			return stream
 
+		def testCheckStockAllActions(self):
+			unittest.F2B.SkipIfFast()
+			import glob
+
+			server = TestServer()
+			transm = server._Server__transm
+
+			for actCfg in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')):
+				act = os.path.basename(actCfg).replace('.conf', '')
+				# transmit artifical jail with each action to the server:
+				stream = self.getDefaultJailStream('j-'+act, act)
+				for cmd in stream:
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
+				# test executing action commands:
+				self._testExecActions(server)
+
+
 		def testCheckStockCommandActions(self):
 			# test cases to check valid ipv4/ipv6 action definition, tuple with (('jail', 'action[params]', 'tests', ...)
 			# where tests is a dictionary contains:
@@ -1347,7 +1402,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# for cmd in stream:
 				# 	print(cmd)
 
-				# filter all start commands (we want not start all jails):
+				# transmit jail to the server:
 				for cmd in stream:
 					# command to server:
 					ret, res = transm.proceed(cmd)

From e0924e0d1bb948141cb4efd494d188436355032e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 21:44:07 +0200
Subject: [PATCH 086/623] test case fix (always sort result of
 `DNSUtils.textToIp`, because order of result from `socket.getaddrinfo` is
 undefined (system depended)

---
 fail2ban/tests/filtertestcase.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 1d4db3a2..4f5dbb55 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1357,9 +1357,11 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
-		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		# sort ipaddr, IPv4 is always smaller as IPv6
+		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
-		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		# sort ipaddr, IPv4 is always smaller as IPv6
+		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1371,7 +1373,8 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		for s in hostnames:
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
-				self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+				# sort ipaddr, IPv4 is always smaller as IPv6
+				self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 

From cb280b817f89386546c58109f2138fea6e0cfc08 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 22:16:35 +0200
Subject: [PATCH 087/623] csocket multi-set fix: prevent to convert `list`,
 `dict`, `set` during transfer (send), this offers a sending of 'multi-set'
 arrays (missed by cherry-picking from multi-set branch)

---
 fail2ban/client/csocket.py     | 4 +++-
 fail2ban/server/server.py      | 4 ++++
 fail2ban/server/transmitter.py | 3 ++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py
index 2e22e5ee..45d58cc6 100644
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@@ -44,7 +44,9 @@ class CSocket:
 	
 	def send(self, msg):
 		# Convert every list member to string
-		obj = dumps([str(m) for m in msg], HIGHEST_PROTOCOL)
+		obj = dumps(map(
+			lambda m: str(m) if not isinstance(m, (list, dict, set)) else m, msg),
+		  HIGHEST_PROTOCOL)
 		self.__csock.send(obj + CSPROTO.END)
 		return self.receive(self.__csock)
 
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 7f75c347..def796a5 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -266,8 +266,10 @@ class Server:
 		flt = self.__jails[name].filter
 		if multiple:
 			for value in value:
+				logSys.debug("  failregex: %r", value)
 				flt.addFailRegex(value)
 		else:
+			logSys.debug("  failregex: %r", value)
 			flt.addFailRegex(value)
 	
 	def delFailRegex(self, name, index):
@@ -280,8 +282,10 @@ class Server:
 		flt = self.__jails[name].filter
 		if multiple:
 			for value in value:
+				logSys.debug("  ignoreregex: %r", value)
 				flt.addIgnoreRegex(value)
 		else:
+			logSys.debug("  ignoreregex: %r", value)
 			flt.addIgnoreRegex(value)
 	
 	def delIgnoreRegex(self, name, index):
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 29d6d189..af2d1b53 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -52,7 +52,7 @@ class Transmitter:
 	
 	def proceed(self, command):
 		# Deserialize object
-		logSys.debug("Command: " + repr(command))
+		logSys.debug("Command: %r", command)
 		try:
 			ret = self.__commandHandler(command)
 			ack = 0, ret
@@ -263,6 +263,7 @@ class Transmitter:
 			action = self.__server.getAction(name, actionname)
 			if multiple:
 				for cmd in command[3]:
+					logSys.debug("  %r", cmd)
 					actionkey = cmd[0]
 					if callable(getattr(action, actionkey, None)):
 						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}

From d4201480552da3e69a6586c271f097ea18b98f41 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 May 2016 22:53:57 +0200
Subject: [PATCH 088/623] database: always explicit convert `ip` to `str`,
 because may be an IPAddr, that will be unsupported type by bind parameter (as
 long as we've found any default wrapper handler for sqlite3)

---
 fail2ban/server/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 6a3d87c3..a6b3feb1 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -455,7 +455,7 @@ class Fail2BanDb(object):
 			queryArgs.append(MyTime.time() - bantime)
 		if ip is not None:
 			query += " AND ip=?"
-			queryArgs.append(ip)
+			queryArgs.append(str(ip))
 		query += " ORDER BY ip, timeofban desc"
 
 		return cur.execute(query, queryArgs)

From 2b6f8737a72703426149dfd42bac4f74b1e8a6ce Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 13 May 2016 17:17:46 -0400
Subject: [PATCH 089/623] ENH: version Fail2Ban in this branch as 0.10.0 alpha
 1

---
 fail2ban/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/version.py b/fail2ban/version.py
index 140ca959..db4daa97 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.9.4.dev0"
+version = "0.10.0a1"

From 7e54cee8d61be84bfb8f3eeff70f9f86b90eef9e Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 21:36:27 -0400
Subject: [PATCH 090/623] updated firewallcmd actions

---
 config/action.d/firewallcmd-allports.conf     | 28 +++-----
 config/action.d/firewallcmd-common.conf       | 66 +++++++++++++++++++
 config/action.d/firewallcmd-ipset.conf        |  6 +-
 config/action.d/firewallcmd-multiport.conf    | 51 +++-----------
 config/action.d/firewallcmd-new.conf          | 29 +++-----
 config/action.d/firewallcmd-rich-logging.conf | 34 +++-------
 config/action.d/firewallcmd-rich-rules.conf   | 35 +++-------
 7 files changed, 119 insertions(+), 130 deletions(-)
 create mode 100644 config/action.d/firewallcmd-common.conf

diff --git a/config/action.d/firewallcmd-allports.conf b/config/action.d/firewallcmd-allports.conf
index 571d5ba6..de0e7f91 100644
--- a/config/action.d/firewallcmd-allports.conf
+++ b/config/action.d/firewallcmd-allports.conf
@@ -6,34 +6,26 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-recidive$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Default name of the chain
-#
-name = default
-
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #
diff --git a/config/action.d/firewallcmd-common.conf b/config/action.d/firewallcmd-common.conf
new file mode 100644
index 00000000..106ef216
--- /dev/null
+++ b/config/action.d/firewallcmd-common.conf
@@ -0,0 +1,66 @@
+# Fail2Ban configuration file
+#
+# Author: Donald Yandt
+#
+
+[Init]
+
+# Option:  name
+# Notes    Default name of the chain
+# Values:  STRING
+name = default
+
+# Option:  family(ipv4)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv4
+
+# Option:  chain
+# Notes    specifies the firewalld chain to which the Fail2Ban rules should be
+#          added
+# Values:  STRING  Default: INPUT_direct
+chain = INPUT_direct
+
+# Option:  zone
+# Notes    use command firewall-cmd --get-active-zones to see a list of all active zones. See firewalld man pages for more information on zones
+# Values:  STRING  Default: public
+zone = public
+
+# Option:  service
+# Notes    use command firewall-cmd --get-services to see a list of services available
+#          Examples zones: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
+#          freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
+#          kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
+#          postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
+#          telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
+# Values:  STRING Default: ssh 
+service = ssh
+
+# Option:  rejecttype (ipv4)
+# Note:    See iptables/firewalld man pages for ipv4 reject types. 
+# Values:  STRING
+rejecttype = icmp-port-unreachable
+
+# Option:  blocktype (ipv4/ipv6)
+# Note:    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
+#          REJECT --reject-with icmp-port-unreachable, DROP
+# Values:  STRING
+blocktype = REJECT --reject-with <rejecttype>
+
+# Option:  rich-blocktype (ipv4/ipv6)
+# Note:    See firewalld man pages for jump targets. Common values are reject, 
+#          reject type="icmp-port-unreachable", drop
+# Values:  STRING
+rich-blocktype = reject type='<rejecttype>'
+
+[Init?family=inet6]
+
+# Option:  family(ipv6)
+# Notes    specifies the socket address family type
+# Values:  STRING  
+family = ipv6
+
+# Option:  rejecttype (ipv6)
+# Note:    See iptables/firewalld man pages for ipv6 reject types. 
+# Values:  STRING
+rejecttype = icmp6-port-unreachable
diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 38b0f3d3..b05f4f53 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -14,14 +14,14 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
 actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
              ipset flush fail2ban-<name>
              ipset destroy fail2ban-<name>
 
diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf
index 438d4cf7..cc40d668 100644
--- a/config/action.d/firewallcmd-multiport.conf
+++ b/config/action.d/firewallcmd-multiport.conf
@@ -5,59 +5,28 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m conntrack --ctstate NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
 # Example actioncheck: firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q '^f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 [Init]
 
-# Default name of the chain
-name = default
-
-chain = INPUT_direct
-
 # Could also use port numbers separated by a comma. 
 port = 1:65535
 
-
-# Option:  protocol
-# Values:  [ tcp | udp | icmp | all ]
-
-protocol = tcp
-
-
-
-# DEV NOTES:
-#
-# Author: Donald Yandt 
-# Uses "FirewallD" instead of the "iptables daemon".
-#
-#
-# Output:
-# actionstart:
-# $ firewall-cmd --direct --add-chain ipv4 filter f2b-apache-modsecurity
-# success
-# $ firewall-cmd --direct --add-rule ipv4 filter f2b-apache-modsecurity 1000 -j RETURN
-# success
-# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 80,443 -j f2b-apache-modsecurity
-# success
-# actioncheck:
-# $ firewall-cmd --direct --get-chains ipv4 filter f2b-apache-modsecurity | sed -e 's, ,\n,g' | grep -q '^f2b-apache-modsecurity$'
-# f2b-apache-modsecurity
-
diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf
index ac72a68a..e64601e1 100644
--- a/config/action.d/firewallcmd-new.conf
+++ b/config/action.d/firewallcmd-new.conf
@@ -4,32 +4,23 @@
 
 [INCLUDES]
 
-before = iptables-common.conf
+before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
-              firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+actionstart = firewall-cmd --direct --add-chain <family> filter f2b-<name>
+              firewall-cmd --direct --add-rule <family> filter f2b-<name> 1000 -j RETURN
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
-             firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
+             firewall-cmd --direct --remove-rules <family> filter f2b-<name>
+             firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | grep -q 'f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | grep -q 'f2b-<name>$'
 
-actionban = firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
+actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
-actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Option:  chain
-# Notes    specifies the iptables chain to which the fail2ban rules should be
-#          added
-# Values:  [ STRING ]
-#
-chain = INPUT_direct
+actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 # DEV NOTES:
 #
diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf
index 1b88c2d9..34cb2413 100644
--- a/config/action.d/firewallcmd-rich-logging.conf
+++ b/config/action.d/firewallcmd-rich-logging.conf
@@ -15,6 +15,10 @@
 # firewall-cmd [--zone=<zone>] --list-all
 # firewall-cmd [--zone=zone] --query-rich-rule='rule'
 
+[INCLUDES]
+
+before = firewallcmd-common.conf
+
 [Definition]
 
 actionstart = 
@@ -26,40 +30,22 @@ actioncheck =
 # you can also use zones and/or service names. 
 #
 # zone example: 
-# 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+#
 # service name example:
-#	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+#
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"; done
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"; done
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done
 
 [Init]
 
-name = default
-
 # log levels are "emerg", "alert", "crit", "error", "warning", "notice", "info" or "debug"
 level = info
 
 # log rate per minute
 rate = 1
 
-zone = public
-
-# use command firewall-cmd --get-services to see a list of services available
-#
-# Examples:
-#
-# amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
-# freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
-# kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
-# postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
-# telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
-
-service = ssh
-
-# reject types: 'icmp-net-unreachable', 'icmp-host-unreachable', 'icmp-port-unreachable', 'icmp-proto-unreachable', 
-# 'icmp-net-prohibited', 'icmp-host-prohibited', 'icmp-admin-prohibited' or 'tcp-reset'
-
-blocktype = reject type='icmp-port-unreachable'
diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf
index 4e39df54..e64c3823 100644
--- a/config/action.d/firewallcmd-rich-rules.conf
+++ b/config/action.d/firewallcmd-rich-rules.conf
@@ -13,6 +13,10 @@
 # firewall-cmd [--zone=<zone>] --list-all
 # firewall-cmd [--zone=zone] --query-rich-rule='rule'
 
+[INCLUDES]
+
+before = firewallcmd-common.conf
+
 [Definition]
 
 actionstart = 
@@ -24,34 +28,15 @@ actioncheck =
 #you can also use zones and/or service names. 
 #
 # zone example: 
-# 	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' port port='<port>' protocol='<protocol>' <rich-blocktype>"
+#
 # service name example:
-#	firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='ipv4' source address='<ip>' service name='<service>' <rich-blocktype>"
+#
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' <blocktype>"; done
+actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
 	   
-actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='<ip>' port port='$p' protocol='<protocol>' <blocktype>"; done
+actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done
 
-[Init]
 
-name = default
-
-zone = public
-
-# use command firewall-cmd --get-services to see a list of services available
-#
-# Examples:
-#
-# amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
-# freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
-# kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
-# postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
-# telnet tftp tftp-client tinc tor-socks transmission-client vdsm vnc-server wbem-https xmpp-bosh xmpp-client xmpp-local xmpp-server
-
-service = ssh
-
-# reject types: 'icmp-net-unreachable', 'icmp-host-unreachable', 'icmp-port-unreachable', 'icmp-proto-unreachable', 
-# 'icmp-net-prohibited', 'icmp-host-prohibited', 'icmp-admin-prohibited' or 'tcp-reset'
-
-blocktype = reject type='icmp-port-unreachable'

From 810d5996b5b96ed637041492a7f80584973fa47b Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:10:25 -0400
Subject: [PATCH 091/623] Update firewallcmd-rich-logging.conf

---
 config/action.d/firewallcmd-rich-logging.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf
index 34cb2413..d2c8fc2f 100644
--- a/config/action.d/firewallcmd-rich-logging.conf
+++ b/config/action.d/firewallcmd-rich-logging.conf
@@ -30,10 +30,10 @@ actioncheck =
 # you can also use zones and/or service names. 
 #
 # zone example: 
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
 #
 # service name example:
-# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <blocktype>"
+# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp 
 

From 07de83e04a453ec5bb172f48db0ba863cf8d42a7 Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:38:10 -0400
Subject: [PATCH 092/623] Update firewallcmd-common.conf

---
 config/action.d/firewallcmd-common.conf | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/config/action.d/firewallcmd-common.conf b/config/action.d/firewallcmd-common.conf
index 106ef216..4abe5318 100644
--- a/config/action.d/firewallcmd-common.conf
+++ b/config/action.d/firewallcmd-common.conf
@@ -10,6 +10,16 @@
 # Values:  STRING
 name = default
 
+# Option   port
+# Notes    Can also use port numbers separated by a comma and in rich-rules comma and/or space.
+# Value    STRING Default: 1:65535
+port = 1:65535
+
+# Option:  protocol
+# Notes    [ tcp | udp | icmp | all ]
+# Values:  STRING Default: tcp
+protocol = tcp
+
 # Option:  family(ipv4)
 # Notes    specifies the socket address family type
 # Values:  STRING  
@@ -28,7 +38,7 @@ zone = public
 
 # Option:  service
 # Notes    use command firewall-cmd --get-services to see a list of services available
-#          Examples zones: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
+#          Examples services: amanda-client amanda-k5-client bacula bacula-client dhcp dhcpv6 dhcpv6-client dns freeipa-ldap freeipa-ldaps 
 #          freeipa-replication ftp high-availability http https imaps ipp ipp-client ipsec iscsi-target kadmin kerberos 
 #          kpasswd ldap ldaps libvirt libvirt-tls mdns mosh mountd ms-wbt mysql nfs ntp openvpn pmcd pmproxy pmwebapi pmwebapis pop3s 
 #          postgresql privoxy proxy-dhcp puppetmaster radius rpc-bind rsyncd samba samba-client sane smtp squid ssh synergy 
@@ -37,18 +47,18 @@ zone = public
 service = ssh
 
 # Option:  rejecttype (ipv4)
-# Note:    See iptables/firewalld man pages for ipv4 reject types. 
+# Notes    See iptables/firewalld man pages for ipv4 reject types. 
 # Values:  STRING
 rejecttype = icmp-port-unreachable
 
 # Option:  blocktype (ipv4/ipv6)
-# Note:    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
+# Notes    See iptables/firewalld man pages for jump targets. Common values are REJECT, 
 #          REJECT --reject-with icmp-port-unreachable, DROP
 # Values:  STRING
 blocktype = REJECT --reject-with <rejecttype>
 
 # Option:  rich-blocktype (ipv4/ipv6)
-# Note:    See firewalld man pages for jump targets. Common values are reject, 
+# Notes    See firewalld man pages for jump targets. Common values are reject, 
 #          reject type="icmp-port-unreachable", drop
 # Values:  STRING
 rich-blocktype = reject type='<rejecttype>'

From ffebde68e0fae17b07e620b8d9bc62aa4c967426 Mon Sep 17 00:00:00 2001
From: TorontoMedia <millertime@live.ca>
Date: Fri, 13 May 2016 22:38:36 -0400
Subject: [PATCH 093/623] Update firewallcmd-multiport.conf

---
 config/action.d/firewallcmd-multiport.conf | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf
index cc40d668..81540e5b 100644
--- a/config/action.d/firewallcmd-multiport.conf
+++ b/config/action.d/firewallcmd-multiport.conf
@@ -24,9 +24,3 @@ actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\
 actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 
 actionunban = firewall-cmd --direct --remove-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
-
-[Init]
-
-# Could also use port numbers separated by a comma. 
-port = 1:65535
-

From 0c44ecfc770113b2e0c2ff3937d3e3a91d7a97ca Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 14 May 2016 15:00:23 +0200
Subject: [PATCH 094/623] action.d/firewallcmd-ipset.conf: different name of
 the match set's for IPv4/IPv6, using conditional <ipmset>, analog to the
 iptables-ipset; test cases for 3 firewallcmd extended;

---
 config/action.d/firewallcmd-ipset.conf |  20 +++--
 fail2ban/tests/servertestcase.py       | 108 +++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 7 deletions(-)

diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index b05f4f53..69447627 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -18,16 +18,16 @@ before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime>
+              firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
-             ipset flush fail2ban-<name>
-             ipset destroy fail2ban-<name>
+actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
-actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
-actionunban = ipset del fail2ban-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -44,6 +44,12 @@ chain = INPUT_direct
 
 bantime = 600
 
+ipmset = f2b-<name>
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+
 
 # DEV NOTES:
 #
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index cc14e97f..1809933c 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1391,6 +1391,114 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
+				# firewallcmd-multiport --
+				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
+					'start': (
+						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`",
+					),
+					'ip4-check': (
+						r"`firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-mp$'`",
+					),
+					'ip6-check': (
+						r"`firewall-cmd --direct --get-chains ipv6 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-mp$'`",
+					),
+					'ip4-ban': (
+						r"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv4 filter f2b-j-w-fwcmd-mp 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv6 filter f2b-j-w-fwcmd-mp 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# firewallcmd-allports --
+				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
+					'start': (
+						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-ap`",
+					),
+					'ip4-check': (
+						r"`firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-ap$'`",
+					),
+					'ip6-check': (
+						r"`firewall-cmd --direct --get-chains ipv6 filter | sed -e 's, ,\n,g' | grep -q '^f2b-j-w-fwcmd-ap$'`",
+					),
+					'ip4-ban': (
+						r"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv4 filter f2b-j-w-fwcmd-ap 0 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`firewall-cmd --direct --remove-rule ipv6 filter f2b-j-w-fwcmd-ap 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# firewallcmd-ipset --
+				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
+					'start': (
+						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset6`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`",
+					),					
+				}),
 			)
 			server = TestServer()
 			transm = server._Server__transm

From 975608dfb629c637622f0d638335637c1c688970 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 15 May 2016 21:08:32 +0200
Subject: [PATCH 095/623] no hardcoded python interpreter path

---
 bin/fail2ban-client                                 | 2 +-
 bin/fail2ban-regex                                  | 2 +-
 bin/fail2ban-server                                 | 2 +-
 bin/fail2ban-testcases                              | 2 +-
 config/filter.d/ignorecommands/apache-fakegooglebot | 2 +-
 fail2ban/client/fail2banregex.py                    | 2 +-
 fail2ban/tests/files/config/apache-auth/digest.py   | 2 +-
 fail2ban/tests/files/ignorecommand.py               | 2 +-
 setup.py                                            | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/bin/fail2ban-client b/bin/fail2ban-client
index bc0c0be8..4b1407c4 100755
--- a/bin/fail2ban-client
+++ b/bin/fail2ban-client
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 
diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex
index 584c1ea7..09044f0a 100755
--- a/bin/fail2ban-regex
+++ b/bin/fail2ban-regex
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/bin/fail2ban-server b/bin/fail2ban-server
index f522f418..5ec645a4 100755
--- a/bin/fail2ban-server
+++ b/bin/fail2ban-server
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 
diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 606b0b06..768c584d 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 """Script to run Fail2Ban tests battery
diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index 86a28eaa..fe4b6591 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
 #
 # Written in Python to reuse built-in Python batteries and not depend on
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 9ecb7229..fae4f354 100755
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py
index 875ebffe..f4fcfcb9 100755
--- a/fail2ban/tests/files/config/apache-auth/digest.py
+++ b/fail2ban/tests/files/config/apache-auth/digest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 import requests
 
 try:
diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py
index dd6b5aab..473980ec 100755
--- a/fail2ban/tests/files/ignorecommand.py
+++ b/fail2ban/tests/files/ignorecommand.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 import sys
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
diff --git a/setup.py b/setup.py
index e3c499d2..5579c981 100755
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 

From 6353de8b0f0d2f130255564e92ed470d5b41822d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 30 Mar 2016 21:50:58 +0200
Subject: [PATCH 096/623] possibility to increase verbosity up to heavy debug
 (command line parameter `-vvv`)

---
 fail2ban/client/fail2bancmdline.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index d49f0e3d..83379a9a 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -165,6 +165,7 @@ class Fail2banCmdLine():
 		return None
 
 	def initCmdLine(self, argv):
+		verbose = 1
 		try:
 			# First time?
 			initial = (self._argv is None)
@@ -232,7 +233,8 @@ class Fail2banCmdLine():
 			return None
 		except Exception as e:
 			output("ERROR: %s" % (e,))
-			#logSys.exception(e)
+			if verbose > 2:
+				logSys.exception(e)
 			return False
 
 	def readConfig(self, jail=None):

From e57321ab1e47aa12fbe4d73273477196c80584cd Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 12 May 2016 09:37:30 -0400
Subject: [PATCH 097/623] BF+ENH:  import signal used in the code, withtmpdir
 -> with_tmpdir (more readable)

case shows how evil except Exception can be ;)
---
 fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++---------
 fail2ban/tests/utils.py                  |  8 +++++++-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 82ffe9b3..53bc41d9 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -28,6 +28,7 @@ import os
 import re
 import sys
 import time
+import signal
 import unittest
 
 from threading import Thread
@@ -38,7 +39,7 @@ from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_s
 from .. import protocol
 from ..server import server
 from ..server.utils import Utils
-from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging
+from .utils import LogCaptureTestCase, logSys, with_tmpdir, shutil, logging
 
 
 STOCK_CONF_DIR = "config"
@@ -252,7 +253,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 			(CLIENT, "-vq", "-V",))
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
 
-	@withtmpdir
+	@with_tmpdir
 	def testClientDump(self, tmp):
 		# use here the stock configuration (if possible)
 		startparams = _start_params(tmp, True)
@@ -261,7 +262,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.assertLogged("Loading files")
 		self.assertLogged("logtarget")
 
-	@withtmpdir
+	@with_tmpdir
 	def testClientStartBackgroundInside(self, tmp):
 		try:
 			# use once the stock configuration (to test starting also)
@@ -300,7 +301,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		finally:
 			_kill_srv(tmp)
 
-	@withtmpdir
+	@with_tmpdir
 	def testClientStartBackgroundCall(self, tmp):
 		try:
 			global INTERACT
@@ -382,7 +383,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		phase['end'] = True
 		logSys.debug("-- end of test worker")
 
-	@withtmpdir
+	@with_tmpdir
 	def testClientStartForeground(self, tmp):
 		th = None
 		try:
@@ -422,7 +423,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 			if th:
 				th.join()
 
-	@withtmpdir
+	@with_tmpdir
 	def testClientFailStart(self, tmp):
 		try:
 			# started directly here, so prevent overwrite test cases logger with "INHERITED"
@@ -483,7 +484,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Usage: " + SERVER)
 		self.assertLogged("Report bugs to ")
 
-	@withtmpdir
+	@with_tmpdir
 	def testServerStartBackground(self, tmp):
 		try:
 			# to prevent fork of test-cases process, start server in background via command:
@@ -523,7 +524,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		phase['end'] = True
 		logSys.debug("-- end of test worker")
 
-	@withtmpdir
+	@with_tmpdir
 	def testServerStartForeground(self, tmp):
 		th = None
 		try:
@@ -563,7 +564,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			if th:
 				th.join()
 
-	@withtmpdir
+	@with_tmpdir
 	def testServerFailStart(self, tmp):
 		try:
 			# started directly here, so prevent overwrite test cases logger with "INHERITED"
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index bfbef4de..be8ce015 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -79,7 +79,12 @@ class F2B(optparse.Values):
 		return wtime
 
 
-def withtmpdir(f):
+def with_tmpdir(f):
+	"""Helper decorator to create a temporary directory
+
+	Directory gets removed after function returns, regardless
+	if exception was thrown of not
+	"""
 	@wraps(f)
 	def wrapper(self, *args, **kwargs):
 		tmp = tempfile.mkdtemp(prefix="f2b-temp")
@@ -90,6 +95,7 @@ def withtmpdir(f):
 			shutil.rmtree(tmp)
 	return wrapper
 
+
 def initTests(opts):
 	unittest.F2B = F2B(opts)
 	# --fast :

From d7ff7d18cdcd94877b7f17fc5634f0331f62232b Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 12 May 2016 09:47:01 -0400
Subject: [PATCH 098/623] RF+ENH: @with_kill_srv fixture to kill_srv in the
 tests

---
 fail2ban/tests/fail2banclienttestcase.py | 362 ++++++++++++-----------
 1 file changed, 183 insertions(+), 179 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 53bc41d9..1a0880c7 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -31,8 +31,10 @@ import time
 import signal
 import unittest
 
+from functools import wraps
 from threading import Thread
 
+
 from ..client import fail2banclient, fail2banserver, fail2bancmdline
 from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait
 from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
@@ -205,6 +207,20 @@ def _kill_srv(pidfile): # pragma: no cover
 			f.close()
 	return True
 
+def with_kill_srv(f):
+	"""Helper to decorate tests which receive in the last argument tmpdir to pass to kill_srv
+
+	To be used in tandem with @with_tmpdir
+	"""
+	@wraps(f)
+	def wrapper(self, *args):
+		pidfile = args[-1]
+		try:
+			return f(self, *args)
+		finally:
+			_kill_srv(pidfile)
+	return wrapper
+
 
 class Fail2banClientServerBase(LogCaptureTestCase):
 
@@ -263,115 +279,111 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.assertLogged("logtarget")
 
 	@with_tmpdir
+	@with_kill_srv
 	def testClientStartBackgroundInside(self, tmp):
+		# use once the stock configuration (to test starting also)
+		startparams = _start_params(tmp, True)
+		# start:
+		self.assertRaises(ExitException, _exec_client, 
+			(CLIENT, "-b") + startparams + ("start",))
+		# wait for server (socket and ready):
+		self._wait_for_srv(tmp, True, startparams=startparams)
+		self.assertLogged("Server ready")
+		self.assertLogged("Exit with code 0")
 		try:
-			# use once the stock configuration (to test starting also)
-			startparams = _start_params(tmp, True)
-			# start:
 			self.assertRaises(ExitException, _exec_client, 
-				(CLIENT, "-b") + startparams + ("start",))
-			# wait for server (socket and ready):
-			self._wait_for_srv(tmp, True, startparams=startparams)
-			self.assertLogged("Server ready")
-			self.assertLogged("Exit with code 0")
-			try:
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-				self.assertRaises(FailExitException, _exec_client, 
-					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
-				self.pruneLog()
-				# start again (should fail):
-				self.assertRaises(FailExitException, _exec_client, 
-					(CLIENT, "-b") + startparams + ("start",))
-				self.assertLogged("Server already running")
-			finally:
-				self.pruneLog()
-				# stop:
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("stop",))
-				self.assertLogged("Shutdown successful")
-				self.assertLogged("Exit with code 0")
-
-			self.pruneLog()
-			# stop again (should fail):
+				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
 			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT,) + startparams + ("stop",))
-			self.assertLogged("Failed to access socket path")
-			self.assertLogged("Is fail2ban running?")
+				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+			self.pruneLog()
+			# start again (should fail):
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT, "-b") + startparams + ("start",))
+			self.assertLogged("Server already running")
 		finally:
-			_kill_srv(tmp)
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
+
+		self.pruneLog()
+		# stop again (should fail):
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT,) + startparams + ("stop",))
+		self.assertLogged("Failed to access socket path")
+		self.assertLogged("Is fail2ban running?")
 
 	@with_tmpdir
+	@with_kill_srv
 	def testClientStartBackgroundCall(self, tmp):
+		global INTERACT
+		startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
+		# start (in new process, using the same python version):
+		cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
+		logSys.debug('Start %s ...', cmd)
+		cmd = cmd + startparams + ("--async", "start",)
+		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+		self.assertTrue(len(ret) and ret[0])
+		# wait for server (socket and ready):
+		self._wait_for_srv(tmp, True, startparams=cmd)
+		self.assertLogged("Server ready")
+		self.pruneLog()
 		try:
-			global INTERACT
-			startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
-			# start (in new process, using the same python version):
-			cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
-			logSys.debug('Start %s ...', cmd)
-			cmd = cmd + startparams + ("--async", "start",)
-			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
-			self.assertTrue(len(ret) and ret[0])
-			# wait for server (socket and ready):
-			self._wait_for_srv(tmp, True, startparams=cmd)
-			self.assertLogged("Server ready")
+			# echo from client (inside):
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertLogged("TEST-ECHO")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+			# interactive client chat with started server:
+			INTERACT += [
+				"echo INTERACT-ECHO",
+				"status",
+				"exit"
+			]
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("-i",))
+			self.assertLogged("INTERACT-ECHO")
+			self.assertLogged("Status", "Number of jail:")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+			# test reload and restart over interactive client:
+			INTERACT += [
+				"reload",
+				"restart",
+				"exit"
+			]
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("-i",))
+			self.assertLogged("Reading config files:")
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Server ready")
+			self.assertLogged("Exit with code 0")
+			self.pruneLog()
+			# test reload missing jail (interactive):
+			INTERACT += [
+				"reload ~~unknown~jail~fail~~",
+				"exit"
+			]
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("-i",))
+			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
+			self.pruneLog()
+			# test reload missing jail (direct):
+			self.assertRaises(FailExitException, _exec_client, 
+				(CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~"))
+			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
+			self.assertLogged("Exit with code -1")
 			self.pruneLog()
-			try:
-				# echo from client (inside):
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-				self.assertLogged("TEST-ECHO")
-				self.assertLogged("Exit with code 0")
-				self.pruneLog()
-				# interactive client chat with started server:
-				INTERACT += [
-					"echo INTERACT-ECHO",
-					"status",
-					"exit"
-				]
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("-i",))
-				self.assertLogged("INTERACT-ECHO")
-				self.assertLogged("Status", "Number of jail:")
-				self.assertLogged("Exit with code 0")
-				self.pruneLog()
-				# test reload and restart over interactive client:
-				INTERACT += [
-					"reload",
-					"restart",
-					"exit"
-				]
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("-i",))
-				self.assertLogged("Reading config files:")
-				self.assertLogged("Shutdown successful")
-				self.assertLogged("Server ready")
-				self.assertLogged("Exit with code 0")
-				self.pruneLog()
-				# test reload missing jail (interactive):
-				INTERACT += [
-					"reload ~~unknown~jail~fail~~",
-					"exit"
-				]
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("-i",))
-				self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
-				self.pruneLog()
-				# test reload missing jail (direct):
-				self.assertRaises(FailExitException, _exec_client, 
-					(CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~"))
-				self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
-				self.assertLogged("Exit with code -1")
-				self.pruneLog()
-			finally:
-				self.pruneLog()
-				# stop:
-				self.assertRaises(ExitException, _exec_client, 
-					(CLIENT,) + startparams + ("stop",))
-				self.assertLogged("Shutdown successful")
-				self.assertLogged("Exit with code 0")
 		finally:
-			_kill_srv(tmp)
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_client, 
+				(CLIENT,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
 
 	def _testClientStartForeground(self, tmp, startparams, phase):
 		# start and wait to end (foreground):
@@ -424,45 +436,42 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				th.join()
 
 	@with_tmpdir
+	@with_kill_srv
 	def testClientFailStart(self, tmp):
-		try:
-			# started directly here, so prevent overwrite test cases logger with "INHERITED"
-			startparams = _start_params(tmp, logtarget="INHERITED")
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
 
-			## wrong config directory
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "--async", "-c", tmp+"/miss", "start",))
-			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
-			self.pruneLog()
+		## wrong config directory
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "--async", "-c", tmp+"/miss", "start",))
+		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+		self.pruneLog()
 
-			## wrong socket
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",))
-			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
-			self.pruneLog()
+		## wrong socket
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",))
+		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+		self.pruneLog()
 
-			## not running
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",))
-			self.assertLogged("Could not find server")
-			self.pruneLog()
+		## not running
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",))
+		self.assertLogged("Could not find server")
+		self.pruneLog()
 
-			## already exists:
-			open(tmp+"/f2b.sock", 'a').close()
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",))
-			self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
-			self.pruneLog()
-			os.remove(tmp+"/f2b.sock")
+		## already exists:
+		open(tmp+"/f2b.sock", 'a').close()
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",))
+		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
+		self.pruneLog()
+		os.remove(tmp+"/f2b.sock")
 
-			## wrong option:
-			self.assertRaises(FailExitException, _exec_client, 
-				(CLIENT, "-s",))
-			self.assertLogged("Usage: ")
-			self.pruneLog()
-
-		finally:
-			_kill_srv(tmp)
+		## wrong option:
+		self.assertRaises(FailExitException, _exec_client, 
+			(CLIENT, "-s",))
+		self.assertLogged("Usage: ")
+		self.pruneLog()
 
 	def testVisualWait(self):
 		sleeptime = 0.035
@@ -485,34 +494,32 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Report bugs to ")
 
 	@with_tmpdir
+	@with_kill_srv
 	def testServerStartBackground(self, tmp):
+		# to prevent fork of test-cases process, start server in background via command:
+		startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
+		# start (in new process, using the same python version):
+		cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
+		logSys.debug('Start %s ...', cmd)
+		cmd = cmd + startparams + ("-b",)
+		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+		self.assertTrue(len(ret) and ret[0])
+		# wait for server (socket and ready):
+		self._wait_for_srv(tmp, True, startparams=cmd)
+		self.assertLogged("Server ready")
+		self.pruneLog()
 		try:
-			# to prevent fork of test-cases process, start server in background via command:
-			startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
-			# start (in new process, using the same python version):
-			cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
-			logSys.debug('Start %s ...', cmd)
-			cmd = cmd + startparams + ("-b",)
-			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
-			self.assertTrue(len(ret) and ret[0])
-			# wait for server (socket and ready):
-			self._wait_for_srv(tmp, True, startparams=cmd)
-			self.assertLogged("Server ready")
-			self.pruneLog()
-			try:
-				self.assertRaises(ExitException, _exec_server, 
-					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
-				self.assertRaises(FailExitException, _exec_server, 
-					(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
-			finally:
-				self.pruneLog()
-				# stop:
-				self.assertRaises(ExitException, _exec_server, 
-					(SERVER,) + startparams + ("stop",))
-				self.assertLogged("Shutdown successful")
-				self.assertLogged("Exit with code 0")
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertRaises(FailExitException, _exec_server, 
+				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
 		finally:
-			_kill_srv(tmp)
+			self.pruneLog()
+			# stop:
+			self.assertRaises(ExitException, _exec_server, 
+				(SERVER,) + startparams + ("stop",))
+			self.assertLogged("Shutdown successful")
+			self.assertLogged("Exit with code 0")
 
 	def _testServerStartForeground(self, tmp, startparams, phase):
 		# start and wait to end (foreground):
@@ -565,30 +572,27 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				th.join()
 
 	@with_tmpdir
+	@with_kill_srv
 	def testServerFailStart(self, tmp):
-		try:
-			# started directly here, so prevent overwrite test cases logger with "INHERITED"
-			startparams = _start_params(tmp, logtarget="INHERITED")
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
 
-			## wrong config directory
-			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER, "-c", tmp+"/miss",))
-			self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
-			self.pruneLog()
+		## wrong config directory
+		self.assertRaises(FailExitException, _exec_server, 
+			(SERVER, "-c", tmp+"/miss",))
+		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+		self.pruneLog()
 
-			## wrong socket
-			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",))
-			self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
-			self.pruneLog()
+		## wrong socket
+		self.assertRaises(FailExitException, _exec_server, 
+			(SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",))
+		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+		self.pruneLog()
 
-			## already exists:
-			open(tmp+"/f2b.sock", 'a').close()
-			self.assertRaises(FailExitException, _exec_server, 
-				(SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",))
-			self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
-			self.pruneLog()
-			os.remove(tmp+"/f2b.sock")
-
-		finally:
-			_kill_srv(tmp)
+		## already exists:
+		open(tmp+"/f2b.sock", 'a').close()
+		self.assertRaises(FailExitException, _exec_server, 
+			(SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",))
+		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
+		self.pruneLog()
+		os.remove(tmp+"/f2b.sock")

From 06dcad76504f82621a11a5156279d02593538d00 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 12:19:29 +0200
Subject: [PATCH 099/623] fixed mixed indentation (spaces through tabs),
 duplicate code removed

---
 bin/fail2ban-testcases  | 4 ++--
 fail2ban/tests/utils.py | 3 ---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 3b18b7c2..7d122039 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -122,7 +122,7 @@ if verbosity > 1: # pragma: no cover
 	if verbosity > 3:
 		fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
 	if verbosity > 2:
-	  fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt
+		fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt
 	else:
 		fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
 #
@@ -134,7 +134,7 @@ logSys.addHandler(stdout)
 #
 if not opts.log_level or opts.log_level != 'critical': # pragma: no cover
 	print("Fail2ban %s test suite. Python %s. Please wait..." \
-	    % (version, str(sys.version).replace('\n', '')))
+			% (version, str(sys.version).replace('\n', '')))
 
 tests = gatherTests(regexps, opts)
 #
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index be8ce015..2d5a38f7 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -418,9 +418,6 @@ class LogCaptureTestCase(unittest.TestCase):
 	def pruneLog(self):
 		self._log.truncate(0)
 
-	def pruneLog(self):
-		self._log.truncate(0)
-
 	def getLog(self):
 		return self._log.getvalue()
 

From 5e06e8be624b1d46169c60797b6959f0c9d7ee49 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 16:42:23 +0200
Subject: [PATCH 100/623] removed shebang and changed mode for fail2banregex.py

---
 fail2ban/client/fail2banregex.py | 1 -
 1 file changed, 1 deletion(-)
 mode change 100755 => 100644 fail2ban/client/fail2banregex.py

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
old mode 100755
new mode 100644
index fae4f354..e2222792
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #

From c6f63c7263741403d24faaf4eb59f716fea205c5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 18:06:46 +0200
Subject: [PATCH 101/623] jailreader: support multiple option groups, syntax
 `action = act[p1=...][p2=...]` + test case for it (see gh-1425, gh-1429)

---
 fail2ban/client/jailreader.py          | 7 ++++++-
 fail2ban/tests/clientreadertestcase.py | 7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index fda5d40c..98086cf4 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -42,9 +42,14 @@ logSys = getLogger(__name__)
 
 class JailReader(ConfigReader):
 	
+	# regex, to extract list of options:
 	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
+	# regex, to iterate over single option in option list, syntax:
+	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
+	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
+	# `action = act[p1=...][p2=...]`
 	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
+		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)')
 	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 5e66e25e..521fda2e 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -255,6 +255,13 @@ class JailReaderTest(LogCaptureTestCase):
 		result = JailReader.extractOptions(option)
 		self.assertEqual(expected, result)
 
+		# And multiple groups (`][` instead of `,`)
+		result = JailReader.extractOptions(option.replace(',', ']['))
+		expected2 = (expected[0],
+		 dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems())
+		)
+		self.assertEqual(expected2, result)
+
 	def testVersionAgent(self):
 		jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR)
 		# emulate jail.read(), because such jail not exists:

From 3e1328c83b2d6fe9b444df1b13a74738a3c78123 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 15 May 2016 22:30:07 +0200
Subject: [PATCH 102/623] split PF config files between all- and multi port

---
 config/action.d/pf-allports.conf              | 73 +++++++++++++++++++
 .../action.d/{pf.conf => pf-multiport.conf}   |  6 --
 2 files changed, 73 insertions(+), 6 deletions(-)
 create mode 100644 config/action.d/pf-allports.conf
 rename config/action.d/{pf.conf => pf-multiport.conf} (95%)

diff --git a/config/action.d/pf-allports.conf b/config/action.d/pf-allports.conf
new file mode 100644
index 00000000..e77bea43
--- /dev/null
+++ b/config/action.d/pf-allports.conf
@@ -0,0 +1,73 @@
+# Fail2Ban configuration file
+#
+# OpenBSD pf ban/unban
+#
+# Author: Nick Hilliard <nick@foobar.org>
+# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
+#
+#
+
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+# we don't enable PF automatically; to enable run pfctl -e 
+# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to any" | pfctl -f-
+
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+# we only disable PF rules we've installed prior
+actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
+             pfctl -t <tablename>-<name> -T flush
+             pfctl -t <tablename>-<name> -T kill
+
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck = pfctl -sr | grep -q <tablename>-<name>
+
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+actionban = pfctl -t <tablename>-<name> -T add <ip>
+
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+#          <failures>  number of failures
+#          <time>  unix timestamp of the ban time
+# Values:  CMD
+#
+# note -r option used to remove matching rule
+actionunban = pfctl -t <tablename>-<name> -T delete <ip>
+
+[Init]
+# Option:  tablename
+# Notes.:  The pf table name.
+# Values:  [ STRING ]
+#
+tablename = f2b
+
+# Option:  protocol
+# Notes.:  internally used by config reader for interpolations.
+# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
+#
+protocol = tcp
+
diff --git a/config/action.d/pf.conf b/config/action.d/pf-multiport.conf
similarity index 95%
rename from config/action.d/pf.conf
rename to config/action.d/pf-multiport.conf
index effd080c..6814a9f3 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf-multiport.conf
@@ -71,9 +71,3 @@ tablename = f2b
 #
 protocol = tcp
 
-# Option:  port
-# Notes.:  the port to block, defaults to any
-# Values:  [ STRING ]
-#
-port = any
-

From e74047ae49807a59d06ccfcc48350c2cab35c59a Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 16:27:48 +0200
Subject: [PATCH 103/623] revert to common config for PF covering multi and
 allports

---
 config/action.d/pf-multiport.conf             | 73 -------------------
 config/action.d/{pf-allports.conf => pf.conf} | 11 ++-
 2 files changed, 10 insertions(+), 74 deletions(-)
 delete mode 100644 config/action.d/pf-multiport.conf
 rename config/action.d/{pf-allports.conf => pf.conf} (91%)

diff --git a/config/action.d/pf-multiport.conf b/config/action.d/pf-multiport.conf
deleted file mode 100644
index 6814a9f3..00000000
--- a/config/action.d/pf-multiport.conf
+++ /dev/null
@@ -1,73 +0,0 @@
-# Fail2Ban configuration file
-#
-# OpenBSD pf ban/unban
-#
-# Author: Nick Hilliard <nick@foobar.org>
-# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
-#
-#
-
-[Definition]
-
-# Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
-# Values:  CMD
-#
-# we don't enable PF automatically; to enable run pfctl -e 
-# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
-actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any port <port>" | pfctl -f-
-
-
-# Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
-# Values:  CMD
-#
-# we only disable PF rules we've installed prior
-actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
-             pfctl -t <tablename>-<name> -T flush
-             pfctl -t <tablename>-<name> -T kill
-
-
-# Option:  actioncheck
-# Notes.:  command executed once before each actionban command
-# Values:  CMD
-#
-actioncheck = pfctl -sr | grep -q <tablename>-<name>
-
-
-# Option:  actionban
-# Notes.:  command executed when banning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
-# Values:  CMD
-#
-actionban = pfctl -t <tablename>-<name> -T add <ip>
-
-
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
-# Values:  CMD
-#
-# note -r option used to remove matching rule
-actionunban = pfctl -t <tablename>-<name> -T delete <ip>
-
-[Init]
-# Option:  tablename
-# Notes.:  The pf table name.
-# Values:  [ STRING ]
-#
-tablename = f2b
-
-# Option:  protocol
-# Notes.:  internally used by config reader for interpolations.
-# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
-#
-protocol = tcp
-
diff --git a/config/action.d/pf-allports.conf b/config/action.d/pf.conf
similarity index 91%
rename from config/action.d/pf-allports.conf
rename to config/action.d/pf.conf
index e77bea43..10b0f328 100644
--- a/config/action.d/pf-allports.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to any <actionoptions>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -71,3 +71,12 @@ tablename = f2b
 #
 protocol = tcp
 
+
+# Option:  port
+# Notes.:  specifies port to monitor
+# Values:  [ NUM | STRING ]  Default:
+#
+port = telnet
+
+actionoptions = 
+multiport = port <port>

From 1e7fd26f5f42fb9eba0fce8bfc54e8e128bae3e7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 May 2016 20:51:12 +0200
Subject: [PATCH 104/623] rename `actionoptions` to `actiontype` in pf-action
 (multiport) + fixed test cases

---
 config/action.d/pf.conf          |  6 +++---
 fail2ban/tests/servertestcase.py | 23 +++++++++++++++++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 10b0f328..1312538e 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any <actionoptions>" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to any<actiontype>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -78,5 +78,5 @@ protocol = tcp
 #
 port = telnet
 
-actionoptions = 
-multiport = port <port>
+actiontype = 
+multiport = <sp>port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 1809933c..ac9927bb 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1372,12 +1372,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
 					),					
 				}),
-				# pf --
+				# pf allports --
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1391,6 +1391,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
+				# pf multiport --
+				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf-mp -T flush`',
+						'`pfctl -t f2b-j-w-pf-mp -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
+				}),
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),

From b5e031f3c3d35d75ea2821ef6a57b191c7796964 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Tue, 17 May 2016 21:19:34 +0200
Subject: [PATCH 105/623] some documentation for multiport use in pf.conf

---
 config/action.d/pf.conf | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 1312538e..5710eb8e 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -78,5 +78,12 @@ protocol = tcp
 #
 port = telnet
 
+# Option: actiontype
+# Notes.: defines additions to the blocking rule
+# Values: leave empty to block all attempts from the host
 actiontype = 
+
+# Option: multiport
+# Notes.: addition to block access only to specific ports
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
 multiport = <sp>port <port>

From 4d51c591c1f3ce39b407d85c2f14ce3a132360fb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 14:50:41 +0200
Subject: [PATCH 106/623] pf.conf: warranted consistently echoing for the pf
 actiontype if actiontype or multiport tags will be customized;

---
 config/action.d/pf.conf | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 5710eb8e..ea70d894 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,7 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to any<actiontype>" | pfctl -f-
+              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -f-
 
 
 # Option:  actionstop
@@ -76,14 +76,20 @@ protocol = tcp
 # Notes.:  specifies port to monitor
 # Values:  [ NUM | STRING ]  Default:
 #
-port = telnet
+#port = telnet
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
 # Values: leave empty to block all attempts from the host
-actiontype = 
+# Default: Value of the allports
+actiontype = any
+
+# Option: allports
+# Notes.: default addition to block all ports
+# Usage.: use in jail config: "banaction = pf[actiontype=<allports>]"
+allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
-multiport = <sp>port <port>
+multiport = any port <port>

From 1ebc3facb1f30c2269f45284aa9072a84be078ba Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 May 2016 10:34:28 -0400
Subject: [PATCH 107/623] BF: maintain previous default beh for pf -- ban a
 port (ssh) only

---
 config/action.d/pf.conf          |  9 ++++-----
 fail2ban/tests/servertestcase.py | 27 +++++++++++++++++++++++----
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index ea70d894..40f4620a 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -75,14 +75,13 @@ protocol = tcp
 # Option:  port
 # Notes.:  specifies port to monitor
 # Values:  [ NUM | STRING ]  Default:
-#
-#port = telnet
+port = ssh
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
 # Values: leave empty to block all attempts from the host
-# Default: Value of the allports
-actiontype = any
+# Default: Value of the multiport
+actiontype = <multiport>
 
 # Option: allports
 # Notes.: default addition to block all ports
@@ -91,5 +90,5 @@ allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
-# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>, port="%(port)s"]"
 multiport = any port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ac9927bb..5fbf8cd6 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1370,14 +1370,14 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),
 					'ip6-unban': (
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
-					),					
+					),
 				}),
-				# pf allports --
+				# pf default - multiport on ssh --
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port ssh" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1391,7 +1391,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
-				# pf multiport --
+				# pf multiport with custom port --
 				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
 					'ip4': (), 'ip6': (),
 					'start': (
@@ -1410,6 +1410,25 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
+				# pf allports --
+				('j-w-pf-ap', 'pf[name=%(__name__)s,actiontype=<allports>]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf-ap -T flush`',
+						'`pfctl -t f2b-j-w-pf-ap -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 2001:db8::`",),
+				}),
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),

From 0fdc56546f8391760a5c7af0ef32749b76f8d091 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 19 May 2016 17:45:41 +0200
Subject: [PATCH 108/623] Fixed misunderstanding of port in (ban)action: port
 will be always specified in jail config ([DEFAULT] or jail)

---
 config/action.d/pf.conf          | 6 +-----
 fail2ban/tests/servertestcase.py | 6 +++---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 40f4620a..b7476fa2 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -72,10 +72,6 @@ tablename = f2b
 protocol = tcp
 
 
-# Option:  port
-# Notes.:  specifies port to monitor
-# Values:  [ NUM | STRING ]  Default:
-port = ssh
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
@@ -90,5 +86,5 @@ allports = any
 
 # Option: multiport
 # Notes.: addition to block access only to specific ports
-# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>, port="%(port)s"]"
+# Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
 multiport = any port <port>
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 5fbf8cd6..fabf77b9 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1372,12 +1372,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
 					),
 				}),
-				# pf default - multiport on ssh --
+				# pf default -- multiport on default port (tag <port> set in jail.conf, but not in this test case)
 				('j-w-pf', 'pf[name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port ssh" | pfctl -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -f-`',
 					),
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
@@ -1411,7 +1411,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
 				# pf allports --
-				('j-w-pf-ap', 'pf[name=%(__name__)s,actiontype=<allports>]', {
+				('j-w-pf-ap', 'pf[actiontype=<allports>][name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',

From 3d99833570d9768c9c2d050fa143f5cae38d7dd4 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 21 May 2016 13:19:49 +0200
Subject: [PATCH 109/623] forgotten convert for `bantime` (str2seconds) in
 action, as long as it should be supplied as parameter of action executable;
 covered via test cases now

---
 fail2ban/server/action.py        |  4 ++--
 fail2ban/tests/servertestcase.py | 24 ++++++++++++------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index ba3d4633..51c5e077 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -229,8 +229,8 @@ class CommandAction(ActionBase):
 	def __setattr__(self, name, value):
 		if not name.startswith('_') and not callable(value):
 			# special case for some pasrameters:
-			if name == 'timeout':
-				value = MyTime.str2seconds(value)
+			if name in ('timeout', 'bantime'):
+				value = str(MyTime.str2seconds(value))
 			# parameters changed - clear properties and substitution cache:
 			self.__properties = None
 			self.__substCache.clear()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ac9927bb..d39847ca 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -638,11 +638,11 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(
 			self.transm.proceed(
 				["set", self.jailName, "action", action, "timeout", "10"]),
-			(0, 10))
+			(0, "10"))
 		self.assertEqual(
 			self.transm.proceed(
 				["get", self.jailName, "action", action, "timeout"]),
-			(0, 10))
+			(0, "10"))
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "delaction", action]),
 			(0, None))
@@ -1125,7 +1125,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			#   etc.
 			testJailsActions = (
 				# iptables-multiport --
-				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
@@ -1163,7 +1163,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-allports --
-				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
@@ -1201,7 +1201,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6 --
-				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
@@ -1233,7 +1233,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6-allports --
-				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
 					'start': (
 						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
@@ -1265,7 +1265,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables --
-				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables`",
@@ -1303,7 +1303,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-new --
-				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'start': (
 						"`iptables -w -N f2b-j-w-iptables-new`",
@@ -1341,7 +1341,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-xt_recent-echo --
-				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
 					'start': (
 						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
@@ -1411,7 +1411,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
 				# firewallcmd-multiport --
-				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
@@ -1449,7 +1449,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-allports --
-				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
@@ -1487,7 +1487,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-ipset --
-				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
 					'start': (
 						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",

From ec7eabbd3ba8dcf02d41afd7511507a089ae2026 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 22 May 2016 00:18:14 +0200
Subject: [PATCH 110/623] correct MANIFEST file

---
 MANIFEST | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST b/MANIFEST
index 11f635c3..8516e63d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -164,7 +164,6 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
-fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
@@ -182,6 +181,7 @@ fail2ban/server/filter.py
 fail2ban/server/filterpyinotify.py
 fail2ban/server/filtersystemd.py
 fail2ban/server/__init__.py
+fail2ban/server/ipdns.py
 fail2ban/server/iso8601.py
 fail2ban/server/jail.py
 fail2ban/server/jails.py

From 9681cd3be8799c35ac1cf5dc69202b5bbd7ff001 Mon Sep 17 00:00:00 2001
From: Alexander Koeppe <format_c@online.de>
Date: Sun, 22 May 2016 10:27:36 +0200
Subject: [PATCH 111/623] remove redundant IPAddr import

---
 fail2ban/server/actions.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 1b72af91..050c12e6 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -42,7 +42,6 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
-from .filter import IPAddr
 from .utils import Utils
 from ..helpers import getLogger
 

From 93f776d2eec1ae962642372e8f522dafc7eebb7d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 25 May 2016 21:33:26 +0200
Subject: [PATCH 112/623] amend for gh-1419: tags substitution bug - wrong
 recognition of cyclic recursion, new test cases covered this

---
 fail2ban/server/action.py        | 28 ++++++++-------
 fail2ban/tests/actiontestcase.py | 62 +++++++++++++++++++++++++++-----
 2 files changed, 69 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 51c5e077..f579c31e 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -46,9 +46,13 @@ _cmd_lock = threading.Lock()
 # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
 allowed_ipv6 = True
 
+# max tag replacement count:
+MAX_TAG_REPLACE_COUNT = 10
+
 # compiled RE for tag name (replacement name) 
 TAG_CRE = re.compile(r'<([^ <>]+)>')
 
+
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
 
@@ -356,32 +360,28 @@ class CommandAction(ActionBase):
 		tags = inptags.copy()
 		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
+		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				if tag in cls._escapedTags:
-					# Escaped so won't match
-					continue
+				# ignore escaped or already done:
+				if tag in done: continue
 				value = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
-				done = {}
-				last_found = tag
+				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
 					#logSys.log(5, 'found: %s' % found_tag)
-					curdone = done.get(last_found)
-					if curdone is None:
-						done[last_found] = curdone = []
-					if found_tag == tag or found_tag in curdone:
+					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 						raise ValueError(
 							"properties contain self referencing definitions "
 							"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-							(tag, found_tag, curdone, value))
+							(tag, found_tag, refCounts, value))
 					repl = None
 					if found_tag not in cls._escapedTags:
 						repl = tags.get(found_tag + '?' + conditional)
@@ -395,8 +395,9 @@ class CommandAction(ActionBase):
 						continue
 					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
-					curdone.append(found_tag)
-					last_found = found_tag
+					# increment reference count:
+					refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+					# the next match for replace:
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
@@ -405,6 +406,9 @@ class CommandAction(ActionBase):
 					if t.search(value):
 						repFlag = True
 					tags[tag] = value
+				# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
+				if '<' not in value: done.add(tag)
+			# stop interpolation, if no replacements anymore:
 			if not repFlag:
 				break
 		return tags
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index ac59a400..457d81bd 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -66,18 +66,62 @@ class CommandActionTest(LogCaptureTestCase):
 			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertRaises(ValueError,
 			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
-		# No-recursion, just multiple replacement of tag <T>, should be successful
-		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
-			self.assertEqual(CommandAction.substituteRecursiveTags(
-				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
+		# We need here an ordered, because the sequence of iteration is very important for this test
+		if OrderedDict:
+			# No cyclic recursion, just multiple replacement of tag <T>, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+					(('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
 				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
 			)
-		# No-recursion, just multiple replacement of tag <T>, should be successful
-		if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test
-			self.assertEqual(CommandAction.substituteRecursiveTags(
-				OrderedDict((('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
-				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
+			# No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+				  (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>')))
+				), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'}
 			)
+			# No cyclic recursion, just multiple replacement of same tags, should be successful:
+			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((
+					('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'),
+					('ipmset', 'f2b-<name>'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables <lockingopt>'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '<multiport>'),
+					('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT',),
+				))
+				), OrderedDict((
+					('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('ipmset', 'f2b-any'),
+					('name', 'any'),
+					('bantime', '600'),
+					('ipsetfamily', 'inet'),
+					('iptables', 'iptables -w'),
+					('lockingopt', '-w'),
+					('chain', 'INPUT'),
+					('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'),
+					('protocol', 'tcp'),
+					('port', 'ssh'),
+					('blocktype', 'REJECT')
+				))
+			)
+			# Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle:
+			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+					('DE', 'cycle <A>'),
+			)) ))
+			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+					('DE', 'cycle <A>'),
+					('A', '<<B><C>>'),
+					('B', 'D'), ('C', 'E'),
+			)) ))
+			
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})

From d3442742716e8253ca592831ae61b51305571fb1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 31 May 2016 21:55:26 +0200
Subject: [PATCH 113/623] separated host match group and tags for ip4, ip6,
 dns, fid (failure-id): - better recognition for usage of textToIp, expected
 or raw value should be used; - separated failure id vs. host (if found use
 `fid` instead of `host` resp. `ip`); - additional optional groups may be used
 in tags replacement by executing actions;

---
 fail2ban/server/failmanager.py    | 26 ++++-----
 fail2ban/server/failregex.py      | 95 +++++++++++++++++++++++-------
 fail2ban/server/filter.py         | 50 +++++++++++++---
 fail2ban/server/ipdns.py          | 96 ++++++++++++++++---------------
 fail2ban/server/ticket.py         | 10 +++-
 fail2ban/tests/samplestestcase.py |  2 +-
 fail2ban/tests/servertestcase.py  | 50 ++++++++++++++++
 7 files changed, 237 insertions(+), 92 deletions(-)

diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py
index b342b280..ee4b049d 100644
--- a/fail2ban/server/failmanager.py
+++ b/fail2ban/server/failmanager.py
@@ -78,9 +78,9 @@ class FailManager:
 	def addFailure(self, ticket, count=1):
 		attempts = 1
 		with self.__lock:
-			ip = ticket.getIP()
+			fid = ticket.getID()
 			try:
-				fData = self.__failList[ip]
+				fData = self.__failList[fid]
 				# if the same object - the same matches but +1 attempt:
 				if fData is ticket:
 					matches = None
@@ -109,7 +109,7 @@ class FailManager:
 					fData = FailTicket(ticket=ticket)
 				if count > ticket.getAttempt():
 					fData.setRetry(count)
-				self.__failList[ip] = fData
+				self.__failList[fid] = fData
 
 			attempts = fData.getRetry()
 			self.__failTotal += 1
@@ -132,7 +132,7 @@ class FailManager:
 	
 	def cleanup(self, time):
 		with self.__lock:
-			todelete = [ip for ip,item in self.__failList.iteritems() \
+			todelete = [fid for fid,item in self.__failList.iteritems() \
 				if item.getLastTime() + self.__maxTime <= time]
 			if len(todelete) == len(self.__failList):
 				# remove all:
@@ -142,27 +142,27 @@ class FailManager:
 				return
 			if len(todelete) / 2.0 <= len(self.__failList) / 3.0:
 				# few as 2/3 should be removed - remove particular items:
-				for ip in todelete:
-					del self.__failList[ip]
+				for fid in todelete:
+					del self.__failList[fid]
 			else:
 				# create new dictionary without items to be deleted:
-				self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \
+				self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \
 					if item.getLastTime() + self.__maxTime > time)
 		self.__bgSvc.service()
 	
-	def delFailure(self, ip):
+	def delFailure(self, fid):
 		with self.__lock:
 			try:
-				del self.__failList[ip]
+				del self.__failList[fid]
 			except KeyError:
 				pass
 	
-	def toBan(self, ip=None):
+	def toBan(self, fid=None):
 		with self.__lock:
-			for ip in ([ip] if ip != None and ip in self.__failList else self.__failList):
-				data = self.__failList[ip]
+			for fid in ([fid] if fid != None and fid in self.__failList else self.__failList):
+				data = self.__failList[fid]
 				if data.getRetry() >= self.__maxRetry:
-					del self.__failList[ip]
+					del self.__failList[fid]
 					return data
 		self.__bgSvc.service()
 		raise FailManagerEmpty
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 6076acc3..d343dbe0 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -62,18 +62,39 @@ class Regex:
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
 
+	##
+	# Replaces "<HOST>", "<IP4>", "<IP6>", "<FID>" with default regular expression for host
+	#
+	# (see gh-1374 for the discussion about other candidates)
+	# @return the replaced regular expression as string
+
 	@staticmethod
 	def _resolveHostTag(regex):
-		# Replace "<HOST>" with default regular expression for host:
-		# Other candidates (see gh-1374 for the discussion about):
-		#   differentiate: r"""(?:(?:::f{4,6}:)?(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<HOST>[\w\-.^_]*\w))"""
-		#     expected many changes in filter, failregex, etc...
-		#   simple:        r"""(?:::f{4,6}:)?(?P<host>[\w\-.^_:]*\w)"""
-		#     not good enough, if not precise expressions around <HOST>, because for example will match '1.2.3.4:23930' as ip-address;
-		# Todo: move this functionality to filter reader, as default <HOST> replacement,
-		#       make it configurable (via jail/filter configs)
-		return regex.replace("<HOST>",
-		 	r"""(?:::f{4,6}:)?(?P<host>(?:\d{1,3}\.){3}\d{1,3}|\[?(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}\]?|(?<=:):)|[\w\-.^_]*\w)""")
+		# 3 groups instead of <HOST> - separated ipv4, ipv6 and host
+		regex = regex.replace("<HOST>",
+		 	r"""(?:(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<dns>[\w\-.^_]*\w))""")
+		# separated ipv4:
+		r = r"""(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})"""
+		regex = regex.replace("<IP4>", r); # self closed
+		regex = regex.replace("<F-IP4/>", r); # closed
+		# separated ipv6:
+		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}?|(?<=:):))"""
+		regex = regex.replace("<IP6>", r); # self closed
+		regex = regex.replace("<F-IP6/>", r); # closed
+		# separated dns:
+		r = r"""(?P<dns>[\w\-.^_]*\w)"""
+		regex = regex.replace("<DNS>", r); # self closed
+		regex = regex.replace("<F-DNS/>", r); # closed
+		# default failure-id as no space tag:
+		regex = regex.replace("<F-ID/>", r"""(?P<fid>\S+)"""); # closed
+		# default failure port, like 80 or http :
+		regex = regex.replace("<F-PORT/>", r"""(?P<port>\w+)"""); # closed
+		# default failure groups (begin / end tag) for customizable expressions:
+		for o,r in (('IP4', 'ip4'), ('IP6', 'ip6'), ('DNS', 'dns'), ('ID', 'fid'), ('PORT', 'fport')):
+			regex = regex.replace("<F-%s>" % o, "(?P<%s>" % r); # open tag
+			regex = regex.replace("</F-%s>" % o, ")"); # close tag
+
+		return regex
 
 	##
 	# Gets the regular expression.
@@ -207,6 +228,13 @@ class RegexException(Exception):
 	pass
 
 
+##
+# Groups used as failure identifier.
+#
+# The order of this tuple is important while searching for failure-id
+#
+FAILURE_ID_GROPS = ("fid", "ip4", "ip6", "dns")
+
 ##
 # Regular expression class.
 #
@@ -220,25 +248,48 @@ class FailRegex(Regex):
 	# Creates a new object. This method can throw RegexException in order to
 	# avoid construction of invalid object.
 	# @param value the regular expression
-	
+
 	def __init__(self, regex):
 		# Initializes the parent.
 		Regex.__init__(self, regex)
-		# Check for group "host"
-		if "host" not in self._regexObj.groupindex:
-			raise RegexException("No 'host' group in '%s'" % self._regex)
+		# Check for group "dns", "ip4", "ip6", "fid"
+		if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]:
+			raise RegexException("No failure-id group in '%s'" % self._regex)
 	
 	##
-	# Returns the matched host.
+	# Returns all matched groups.
 	#
-	# This corresponds to the pattern matched by the named group "host".
-	# @return the matched host
+
+	def getGroups(self):
+		return self._matchCache.groupdict()
+
+	##
+	# Returns the matched failure id.
+	#
+	# This corresponds to the pattern matched by the named group from given groups.
+	# @return the matched failure-id
 	
-	def getHost(self):
-		host = self._matchCache.group("host")
-		if host is None:
+	def getFailID(self, groups=FAILURE_ID_GROPS):
+		fid = None
+		for grp in groups:
+			try:
+				fid = self._matchCache.group(grp)
+			except IndexError:
+				continue
+			if fid is not None:
+				break
+		if fid is None:
 			# Gets a few information.
 			s = self._matchCache.string
 			r = self._matchCache.re
-			raise RegexException("No 'host' found in '%s' using '%s'" % (s, r))
-		return str(host)
+			raise RegexException("No group found in '%s' using '%s'" % (s, r))
+		return str(fid)
+
+	##
+	# Returns the matched host.
+	#
+	# This corresponds to the pattern matched by the named group "ip4", "ip6" or "dns".
+	# @return the matched host
+	
+	def getHost(self):
+		return self.getFailID(("ip4", "ip6", "dns"))
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c1cd2d09..d0d8f74e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -418,6 +418,9 @@ class Filter(JailThread):
 			ip = element[1]
 			unixTime = element[2]
 			lines = element[3]
+			fail = {}
+			if len(element) > 4:
+				fail = element[4]
 			logSys.debug("Processing line with time:%s and ip:%s", 
 					unixTime, ip)
 			if unixTime < MyTime.time() - self.getFindTime():
@@ -429,7 +432,7 @@ class Filter(JailThread):
 			logSys.info(
 				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 			)
-			tick = FailTicket(ip, unixTime, lines)
+			tick = FailTicket(ip, unixTime, lines, data=fail)
 			self.failManager.addFailure(tick)
 
 	##
@@ -457,7 +460,12 @@ class Filter(JailThread):
 		checkAllRegex=False):
 		failList = list()
 
-		# Checks if we must ignore this line.
+		cidr = IPAddr.CIDR_UNSPEC
+		if self.__useDns == "raw":
+			returnRawHost = True
+			cidr = IPAddr.CIDR_RAW
+
+		# Checks if we mut ignore this line.
 		if self.ignoreLine([tupleLine[::2]]) is not None:
 			# The ignoreregex matched. Return.
 			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
@@ -518,19 +526,45 @@ class Filter(JailThread):
 						 % ("\n".join(failRegex.getMatchedLines()), timeText))
 				else:
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+					# retrieve failure-id, host, etc from failure match:
+					raw = returnRawHost
 					try:
-						host = failRegex.getHost()
-						if returnRawHost or self.__useDns == "raw":
-							failList.append([failRegexIndex, IPAddr(host), date,
-								 failRegex.getMatchedLines()])
+						fail = failRegex.getGroups()
+						# failure-id:
+						fid = fail.get('fid')
+						# ip-address or host:
+						host = fail.get('ip4')
+						if host is not None:
+							raw = True
+						else:
+							host = fail.get('ip6')
+							if host is not None:
+								raw = True
+							else:
+								host = fail.get('dns')
+								if host is None:
+									# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
+									if fid is None:
+										fid = failRegex.getFailID()
+									host = fid
+									cidr = IPAddr.CIDR_RAW
+						# if raw - add single ip or failure-id,
+						# otherwise expand host to multiple ips using dns (or ignore it if not valid):
+						if raw:
+							ip = IPAddr(host, cidr)
+							# check host equal failure-id, if not - failure with complex id:
+							if fid is not None and fid != host:
+								ip = IPAddr(fid, IPAddr.CIDR_RAW)
+							failList.append([failRegexIndex, ip, date,
+								failRegex.getMatchedLines(), fail])
 							if not checkAllRegex:
 								break
 						else:
 							ips = DNSUtils.textToIp(host, self.__useDns)
 							if ips:
 								for ip in ips:
-									failList.append([failRegexIndex, ip, 
-										 date, failRegex.getMatchedLines()])
+									failList.append([failRegexIndex, ip, date,
+										failRegex.getMatchedLines(), fail])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 8109c21a..61c1ba68 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -138,18 +138,21 @@ class IPAddr(object):
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
 
-	def __new__(cls, ipstr, cidr=-1):
+	CIDR_RAW = -2
+	CIDR_UNSPEC = -1
+
+	def __new__(cls, ipstr, cidr=CIDR_UNSPEC):
 		# check already cached as IPAddr
 		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
 		if ip is not None:
 			return ip
 		# wrap mask to cidr (correct plen):
-		if cidr == -1:
+		if cidr == IPAddr.CIDR_UNSPEC:
 			ipstr, cidr = IPAddr.__wrap_ipstr(ipstr)
 			args = (ipstr, cidr)
 			# check cache again:
-			if cidr != -1:
+			if cidr != IPAddr.CIDR_UNSPEC:
 				ip = IPAddr.CACHE_OBJ.get(args)
 				if ip is not None:
 					return ip
@@ -166,7 +169,7 @@ class IPAddr(object):
 			ipstr = ipstr[1:-1]
 		# test mask:
 		if "/" not in ipstr:
-			return ipstr, -1
+			return ipstr, IPAddr.CIDR_UNSPEC
 		s = ipstr.split('/', 1)
 		# IP address without CIDR mask
 		if len(s) > 2:
@@ -176,7 +179,7 @@ class IPAddr(object):
 		s[1] = long(s[1])
 		return s
 		
-	def __init(self, ipstr, cidr=-1):
+	def __init(self, ipstr, cidr=CIDR_UNSPEC):
 		""" initialize IP object by converting IP address string
 			to binary to integer
 		"""
@@ -184,49 +187,48 @@ class IPAddr(object):
 		self._addr = 0
 		self._plen = 0
 		self._maskplen = None
-		self._raw = ""
+		# always save raw value (normally used if really raw or not valid only):
+		self._raw = ipstr
+		# if not raw - recognize family, set addr, etc.:
+		if cidr != IPAddr.CIDR_RAW:
+			for family in [socket.AF_INET, socket.AF_INET6]:
+				try:
+					binary = socket.inet_pton(family, ipstr)
+					self._family = family
+					break
+				except socket.error:
+					continue
 
-		for family in [socket.AF_INET, socket.AF_INET6]:
-			try:
-				binary = socket.inet_pton(family, ipstr)
-				self._family = family
-				break
-			except socket.error:
-				continue
-
-		if self._family == socket.AF_INET:
-			# convert host to network byte order
-			self._addr, = struct.unpack("!L", binary)
-			self._plen = 32
-
-			# mask out host portion if prefix length is supplied
-			if cidr is not None and cidr >= 0:
-				mask = ~(0xFFFFFFFFL >> cidr)
-				self._addr &= mask
-				self._plen = cidr
-
-		elif self._family == socket.AF_INET6:
-			# convert host to network byte order
-			hi, lo = struct.unpack("!QQ", binary)
-			self._addr = (hi << 64) | lo
-			self._plen = 128
-
-			# mask out host portion if prefix length is supplied
-			if cidr is not None and cidr >= 0:
-				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
-				self._addr &= mask
-				self._plen = cidr
-
-			# if IPv6 address is a IPv4-compatible, make instance a IPv4
-			elif self.isInNet(IPAddr.IP6_4COMPAT):
-				self._addr = lo & 0xFFFFFFFFL
-				self._family = socket.AF_INET
+			if self._family == socket.AF_INET:
+				# convert host to network byte order
+				self._addr, = struct.unpack("!L", binary)
 				self._plen = 32
+
+				# mask out host portion if prefix length is supplied
+				if cidr is not None and cidr >= 0:
+					mask = ~(0xFFFFFFFFL >> cidr)
+					self._addr &= mask
+					self._plen = cidr
+
+			elif self._family == socket.AF_INET6:
+				# convert host to network byte order
+				hi, lo = struct.unpack("!QQ", binary)
+				self._addr = (hi << 64) | lo
+				self._plen = 128
+
+				# mask out host portion if prefix length is supplied
+				if cidr is not None and cidr >= 0:
+					mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+					self._addr &= mask
+					self._plen = cidr
+
+				# if IPv6 address is a IPv4-compatible, make instance a IPv4
+				elif self.isInNet(IPAddr.IP6_4COMPAT):
+					self._addr = lo & 0xFFFFFFFFL
+					self._family = socket.AF_INET
+					self._plen = 32
 		else:
-			# string couldn't be converted neither to a IPv4 nor
-			# to a IPv6 address - retain raw input for later use
-			# (e.g. DNS resolution)
-			self._raw = ipstr
+			self._family = IPAddr.CIDR_RAW
 
 	def __repr__(self):
 		return self.ntoa
@@ -270,6 +272,8 @@ class IPAddr(object):
 		return self._family != socket.AF_UNSPEC
 
 	def __eq__(self, other):
+		if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr):
+			return self._raw == other
 		if not isinstance(other, IPAddr):
 			if other is None: return False
 			other = IPAddr(other)
@@ -285,6 +289,8 @@ class IPAddr(object):
 		return not (self == other)
 
 	def __lt__(self, other):
+		if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr):
+			return self._raw < other
 		if not isinstance(other, IPAddr):
 			if other is None: return False
 			other = IPAddr(other)
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 130de4f2..65ed83c3 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -36,7 +36,7 @@ logSys = getLogger(__name__)
 
 class Ticket:
 	
-	def __init__(self, ip=None, time=None, matches=None, ticket=None):
+	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		"""Ticket constructor
 
 		@param ip the IP address
@@ -50,6 +50,7 @@ class Ticket:
 		self._banTime = None;
 		self._time = time if time is not None else MyTime.time()
 		self._data = {'matches': [], 'failures': 0}
+		self._data.update(data)
 		if ticket:
 			# ticket available - copy whole information from ticket:
 			self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__)
@@ -78,6 +79,9 @@ class Ticket:
 			value = IPAddr(value)
 		self.__ip = value
 	
+	def getID(self):
+		return self._data.get('fid', self.__ip)
+	
 	def getIP(self):
 		return self.__ip
 	
@@ -164,12 +168,12 @@ class Ticket:
 
 class FailTicket(Ticket):
 
-	def __init__(self, ip=None, time=None, matches=None, ticket=None):
+	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		# this class variables:
 		self.__retry = 0
 		self.__lastReset = None
 		# create/copy using default ticket constructor:
-		Ticket.__init__(self, ip, time, matches, ticket)
+		Ticket.__init__(self, ip, time, matches, data, ticket)
 		# init:
 		if ticket is None:
 			self.__lastReset = time if time is not None else self.getTime()
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 326d09f7..7af12f13 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -127,7 +127,7 @@ def testSampleRegexsFactory(name, basedir):
 								 (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
 
 				# Verify timestamp and host as expected
-				failregex, host, fail2banTime, lines = ret[0]
+				failregex, host, fail2banTime, lines, fail = ret[0]
 				self.assertEqual(host, faildata.get("host", None))
 
 				t = faildata.get("time", None)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 7557a501..9eb88c21 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -933,6 +933,14 @@ class RegexTests(unittest.TestCase):
 
 	def testHost(self):
 		self.assertRaises(RegexException, FailRegex, '')
+		self.assertRaises(RegexException, FailRegex, '^test no group$')
+		self.assertTrue(FailRegex('^test <HOST> group$'))
+		self.assertTrue(FailRegex('^test <IP4> group$'))
+		self.assertTrue(FailRegex('^test <IP6> group$'))
+		self.assertTrue(FailRegex('^test <DNS> group$'))
+		self.assertTrue(FailRegex('^test id group: ip:port = <F-ID><IP4>(?::<F-PORT/>)?</F-ID>$'))
+		self.assertTrue(FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$'))
+		self.assertTrue(FailRegex('^test id group: anything = <F-ID/>$'))
 		# Testing obscure case when host group might be missing in the matched pattern,
 		# e.g. if we made it optional.
 		fr = FailRegex('%%<HOST>?')
@@ -940,6 +948,30 @@ class RegexTests(unittest.TestCase):
 		fr.search([('%%',"","")])
 		self.assertTrue(fr.hasMatched())
 		self.assertRaises(RegexException, fr.getHost)
+		# The same as above but using separated IPv4/IPv6 expressions
+		fr = FailRegex('%%inet(?:=<F-IP4/>|inet6=<F-IP6/>)?')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('%%inet=test',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertRaises(RegexException, fr.getHost)
+		# Success case: using separated IPv4/IPv6 expressions (no HOST)
+		fr = FailRegex('%%(?:inet(?:=<IP4>|6=<IP6>)?|dns=<DNS>?)')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('%%inet=192.0.2.1',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), '192.0.2.1')
+		fr.search([('%%inet6=2001:DB8::',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), '2001:DB8::')
+		fr.search([('%%dns=example.com',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getHost(), 'example.com')
+		# Success case: using user as failure-id
+		fr = FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$')
+		self.assertFalse(fr.hasMatched())
+		fr.search([('test id group: user:(test login name)',"","")])
+		self.assertTrue(fr.hasMatched())
+		self.assertEqual(fr.getFailID(), 'test login name')
 
 
 class _BadThread(JailThread):
@@ -998,6 +1030,24 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
 		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
 
+	def test_IPAddr_Raw(self):
+		# raw string:
+		r = IPAddr('xxx', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, 'xxx')
+		self.assertEqual('xxx', str(r))
+		self.assertNotEqual(r, IPAddr('xxx'))
+		# raw (not IP, for example host:port as string):
+		r = IPAddr('1:2', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, '1:2')
+		self.assertEqual('1:2', str(r))
+		self.assertNotEqual(r, IPAddr('1:2'))
+
 	def _testExecActions(self, server):
 		jails = server._Server__jails
 		for jail in jails:

From e40a8c8ae832504107d2ebdd69903e516c89ca7f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 11:22:14 +0200
Subject: [PATCH 114/623] small code review

---
 fail2ban/server/filter.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index d0d8f74e..318d8e49 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -533,21 +533,17 @@ class Filter(JailThread):
 						# failure-id:
 						fid = fail.get('fid')
 						# ip-address or host:
-						host = fail.get('ip4')
+						host = fail.get('ip4') or fail.get('ip6')
 						if host is not None:
 							raw = True
 						else:
-							host = fail.get('ip6')
-							if host is not None:
-								raw = True
-							else:
-								host = fail.get('dns')
-								if host is None:
-									# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
-									if fid is None:
-										fid = failRegex.getFailID()
-									host = fid
-									cidr = IPAddr.CIDR_RAW
+							host = fail.get('dns')
+							if host is None:
+								# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
+								if fid is None:
+									fid = failRegex.getFailID()
+								host = fid
+								cidr = IPAddr.CIDR_RAW
 						# if raw - add single ip or failure-id,
 						# otherwise expand host to multiple ips using dns (or ignore it if not valid):
 						if raw:

From 8893473d82f3732cfec6c7e302e1b94ff6b62b99 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 11:47:24 +0200
Subject: [PATCH 115/623] pypy fix: KeyError instead of IndexError by missing
 group

---
 fail2ban/server/failregex.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index d343dbe0..12b8986a 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -274,7 +274,7 @@ class FailRegex(Regex):
 		for grp in groups:
 			try:
 				fid = self._matchCache.group(grp)
-			except IndexError:
+			except (IndexError, KeyError):
 				continue
 			if fid is not None:
 				break

From e39126f6301fdf86348f43283b802dc28ad2a76f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 13:15:46 +0200
Subject: [PATCH 116/623] badip timeout option introduced, set to 30 seconds in
 our test cases

---
 config/action.d/badips.py              | 11 +++++++----
 fail2ban/tests/action_d/test_badips.py |  1 +
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/config/action.d/badips.py b/config/action.d/badips.py
index 025289ca..4bc879a1 100644
--- a/config/action.d/badips.py
+++ b/config/action.d/badips.py
@@ -80,14 +80,17 @@ class BadIPsAction(ActionBase):
 		If invalid `category`, `score`, `banaction` or `updateperiod`.
 	"""
 
+	TIMEOUT = 10
 	_badips = "http://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"):
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
+		self.timeout = timeout
 		self.agent = agent
 		self.category = category
 		self.score = score
@@ -119,7 +122,7 @@ class BadIPsAction(ActionBase):
 		"""
 		try:
 			response = urlopen(
-				self._Request("/".join([self._badips, "get", "categories"])), None, 3)
+				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -173,7 +176,7 @@ class BadIPsAction(ActionBase):
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
@@ -358,7 +361,7 @@ class BadIPsAction(ActionBase):
 			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
-			response = urlopen(self._Request(url))
+			response = urlopen(self._Request(url), timeout=self.timeout)
 		except HTTPError as response:
 			messages = json.loads(response.read().decode('utf-8'))
 			self._logSys.error(
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 74594420..97dabada 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -39,6 +39,7 @@ if sys.version_info >= (2,7):
 			self.jail.actions.add("badips", pythonModule, initOpts={
 				'category': "ssh",
 				'banaction': "test",
+				'timeout': (3 if unittest.F2B.fast else 30),
 				})
 			self.action = self.jail.actions["badips"]
 

From 2efcf3c17be9414a8fdbb9f748b38a1b260bd7be Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Jun 2016 13:17:18 +0200
Subject: [PATCH 117/623] increase of test coverage (+ test_IPAddr moved to one
 place), small bugs fixed

---
 fail2ban/server/ipdns.py         |  7 +--
 fail2ban/tests/filtertestcase.py | 84 ++++++++++++++++++++++++++++++++
 fail2ban/tests/servertestcase.py | 22 ---------
 3 files changed, 86 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 61c1ba68..cca63535 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -85,10 +85,7 @@ class DNSUtils:
 			return v
 		# retrieve name
 		try:
-			if not isinstance(ip, IPAddr):
-				v = socket.gethostbyaddr(ip)[0]
-			else:
-				v = socket.gethostbyaddr(ip.ntoa)[0]
+			v = socket.gethostbyaddr(ip)[0]
 		except socket.error, e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
@@ -359,7 +356,7 @@ class IPAddr(object):
 			if not suffix:
 				suffix = "in-addr.arpa."
 		elif self.isIPv6:
-			exploded_ip = self.hexdump()
+			exploded_ip = self.hexdump
 			if not suffix:
 				suffix = "ip6.arpa."
 		else:
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 4f5dbb55..dd3468ed 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1353,6 +1353,42 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
 
+	def test_IPAddr(self):
+		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
+		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
+
+	def test_IPAddr_Raw(self):
+		# raw string:
+		r = IPAddr('xxx', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, 'xxx')
+		self.assertEqual('xxx', str(r))
+		self.assertNotEqual(r, IPAddr('xxx'))
+		# raw (not IP, for example host:port as string):
+		r = IPAddr('1:2', IPAddr.CIDR_RAW)
+		self.assertFalse(r.isIPv4)
+		self.assertFalse(r.isIPv6)
+		self.assertTrue(r.isValid)
+		self.assertEqual(r, '1:2')
+		self.assertEqual('1:2', str(r))
+		self.assertNotEqual(r, IPAddr('1:2'))
+		# raw vs ip4 (raw is not an ip):
+		r = IPAddr('93.184.0.1', IPAddr.CIDR_RAW)
+		ip4 = IPAddr('93.184.0.1')
+		self.assertNotEqual(ip4, r)
+		self.assertNotEqual(r, ip4)
+		self.assertTrue(r < ip4)
+		self.assertTrue(r < ip4)
+		# raw vs ip6 (raw is not an ip):
+		r = IPAddr('1::2', IPAddr.CIDR_RAW)
+		ip6 = IPAddr('1::2')
+		self.assertNotEqual(ip6, r)
+		self.assertNotEqual(r, ip6)
+		self.assertTrue(r < ip6)
+		self.assertTrue(r < ip6)
+
 	def testUseDns(self):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
@@ -1377,14 +1413,25 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 				self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
+		# pure ips:
+		for s in ('93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'):
+			ips = DNSUtils.textToIp(s, 'yes')
+			self.assertEqual(ips, [s])
+			self.assertTrue(isinstance(ips[0], IPAddr))
 
 	def testIpToName(self):
 		unittest.F2B.SkipIfNoNetwork()
 		res = DNSUtils.ipToName('8.8.4.4')
 		self.assertEqual(res, 'google-public-dns-b.google.com')
+		# same as above, but with IPAddr:
+		res = DNSUtils.ipToName(IPAddr('8.8.4.4'))
+		self.assertEqual(res, 'google-public-dns-b.google.com')
 		# invalid ip (TEST-NET-1 according to RFC 5737)
 		res = DNSUtils.ipToName('192.0.2.0')
 		self.assertEqual(res, None)
+		# invalid ip:
+		res = DNSUtils.ipToName('192.0.2.888')
+		self.assertEqual(res, None)
 
 	def testAddr2bin(self):
 		res = IPAddr('10.0.0.0')
@@ -1398,11 +1445,44 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = IPAddr('10.0.0.1', cidr=31L)
 		self.assertEqual(res.addr, 167772160L)
 
+		self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000')
+		self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002')
+		self.assertEqual(IPAddr('xxx').hexdump, '')
+
+		self.assertEqual(IPAddr('192.0.2.0').getPTR(), '0.2.0.192.in-addr.arpa.')
+		self.assertEqual(IPAddr('192.0.2.1').getPTR(), '1.2.0.192.in-addr.arpa.')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:1946').getPTR(), 
+			'6.4.9.1.8.c.5.2.3.9.8.1.8.4.2.0.1.0.0.0.0.2.2.0.0.0.8.2.6.0.6.2.ip6.arpa.')
+
 	def testIPAddr_Equal6(self):
 		self.assertEqual(
 			IPAddr('2606:2800:220:1:248:1893::'),
 			IPAddr('2606:2800:220:1:248:1893:0:0')
 		)
+		# special case IPv6 in brackets:
+		self.assertEqual(
+			IPAddr('[2606:2800:220:1:248:1893::]'),
+			IPAddr('2606:2800:220:1:248:1893:0:0')
+		)
+
+	def testIPAddr_InInet(self):
+		ip4net = IPAddr('93.184.0.1/24')
+		ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120')
+		# ip4:
+		self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net))
+		self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net))
+		self.assertFalse(IPAddr('93.184.1.0').isInNet(ip4net))
+		self.assertFalse(IPAddr('93.184.0.1').isInNet(ip6net))
+		# ip6:
+		self.assertTrue(IPAddr('2606:2800:220:1:248:1893:25c8:1').isInNet(ip6net))
+		self.assertTrue(IPAddr('2606:2800:220:1:248:1893:25c8:ff').isInNet(ip6net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:100').isInNet(ip6net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:100').isInNet(ip4net))
+		# raw not in net:
+		self.assertFalse(IPAddr('93.184.0.1', IPAddr.CIDR_RAW).isInNet(ip4net))
+		self.assertFalse(IPAddr('2606:2800:220:1:248:1893:25c8:1', IPAddr.CIDR_RAW).isInNet(ip6net))
+		# invalid not in net:
+		self.assertFalse(IPAddr('xxx').isInNet(ip4net))
 
 	def testIPAddr_Compare(self):
 		ip4 = [
@@ -1473,6 +1553,10 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(str(IPAddr('2606:28ff:220:1:248:1893:25c8::/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')), 
 			'2606:28ff:220:1:248:1893:25c8:0')
 
+	def testIPAddr_CIDR_Wrong(self):
+		# too many plen representations:
+		self.assertRaises(ValueError, IPAddr, '2606:28ff:220:1:248:1893:25c8::/ffff::/::1')
+
 	def testIPAddr_CIDR_Repr(self):
 		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
 			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 9eb88c21..48b0b255 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1026,28 +1026,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				logSys.debug(l)
 		return True
 
-	def test_IPAddr(self):
-		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
-		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
-
-	def test_IPAddr_Raw(self):
-		# raw string:
-		r = IPAddr('xxx', IPAddr.CIDR_RAW)
-		self.assertFalse(r.isIPv4)
-		self.assertFalse(r.isIPv6)
-		self.assertTrue(r.isValid)
-		self.assertEqual(r, 'xxx')
-		self.assertEqual('xxx', str(r))
-		self.assertNotEqual(r, IPAddr('xxx'))
-		# raw (not IP, for example host:port as string):
-		r = IPAddr('1:2', IPAddr.CIDR_RAW)
-		self.assertFalse(r.isIPv4)
-		self.assertFalse(r.isIPv6)
-		self.assertTrue(r.isValid)
-		self.assertEqual(r, '1:2')
-		self.assertEqual('1:2', str(r))
-		self.assertNotEqual(r, IPAddr('1:2'))
-
 	def _testExecActions(self, server):
 		jails = server._Server__jails
 		for jail in jails:

From 4180cc362e63e49c566a8edd178ec21fa68a2875 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 11 Jul 2016 13:22:58 +0200
Subject: [PATCH 118/623] code review

---
 fail2ban/client/fail2banclient.py  | 12 ++++++++----
 fail2ban/client/fail2bancmdline.py |  5 +----
 fail2ban/protocol.py               |  3 +++
 fail2ban/server/server.py          |  2 +-
 fail2ban/tests/utils.py            |  7 +++----
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index a8e0a331..e3797279 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -411,11 +411,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				signal.signal(s, sh)
 
 
-##
-# Wonderful visual :)
-#
-
 class _VisualWait:
+	"""Small progress indication (as "wonderful visual") during waiting process
+	"""
 	pos = 0
 	delta = 1
 	def __init__(self, maxpos=10):
@@ -427,6 +425,8 @@ class _VisualWait:
 			sys.stdout.write('\r'+(' '*(35+self.maxpos))+'\r')
 			sys.stdout.flush()
 	def heartbeat(self):
+		"""Show or step for progress indicator
+		"""
 		if not self.pos:
 			sys.stdout.write("\nINFO   [#" + (' '*self.maxpos) + "] Waiting on the server...\r\x1b[8C")
 		self.pos += self.delta
@@ -441,6 +441,8 @@ class _VisualWait:
 		elif self.pos < 2:
 			self.delta = 1
 class _NotVisualWait:
+	"""Mockup for invisible progress indication (not verbose)
+	"""
 	def __enter__(self):
 		return self
 	def __exit__(self, *args):
@@ -449,6 +451,8 @@ class _NotVisualWait:
 		pass
 
 def VisualWait(verbose, *args, **kwargs):
+	"""Wonderful visual progress indication (if verbose)
+	"""
 	return _VisualWait(*args, **kwargs) if verbose > 1 else _NotVisualWait()
 
 
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 83379a9a..9110a2b8 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -83,9 +83,6 @@ class Fail2banCmdLine():
 		output("Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors")
 		output("Copyright of modifications held by their respective authors.")
 		output("Licensed under the GNU General Public License v2 (GPL).")
-		output("")
-		output("Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>.")
-		output("Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>.")
 
 	def dispUsage(self):
 		""" Prints Fail2Ban command line options and exits
@@ -262,7 +259,7 @@ class Fail2banCmdLine():
 	@staticmethod
 	def exit(code=0): # pragma: no cover - can't test
 		logSys.debug("Exit with code %s", code)
-		if os._exit:
+		if hasattr(os, '_exit') and os._exit:
 			os._exit(code)
 		else:
 			sys.exit(code)
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 648666a1..d671f3c3 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -27,6 +27,9 @@ __license__ = "GPL"
 import textwrap
 
 def output(s):
+	"""Default output handler for printing protocol. 
+  Used to ease mocking in the test cases.
+	"""
 	print(s)
 
 ##
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 4a2a7c11..bade12cb 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -419,7 +419,7 @@ class Server:
 				getLogger("fail2ban").setLevel(getattr(logging, value))
 				self.__logLevel = value
 			except AttributeError:
-				raise ValueError("Invalid log level")
+				raise ValueError("Invalid log level %r" % value)
 	
 	##
 	# Get the logging level.
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 2d5a38f7..a06fbef0 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -54,10 +54,9 @@ if not CONFIG_DIR:
 	else:
 		CONFIG_DIR = '/etc/fail2ban'
 
-# In not installed env (setup, test-cases) use fail2ban modules from main directory:
-if 1 or os.environ.get('PYTHONPATH', None) is None:
-	os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
-		os.path.abspath(__file__)))))
+# During the test cases (or setup) use fail2ban modules from main directory:
+os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
+	os.path.abspath(__file__)))))
 
 class F2B(optparse.Values):
 	def __init__(self, opts={}):

From 6a2b70f11db9841495c384f30e53941714ebd18a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 11 Jul 2016 13:26:10 +0200
Subject: [PATCH 119/623] os.path.join instead of "/" - concatenation

---
 fail2ban/tests/fail2banclienttestcase.py | 62 ++++++++++++------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 1a0880c7..ed961033 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -111,24 +111,24 @@ def _out_file(fn): # pragma: no cover
 	logSys.debug('-'*30)
 
 def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
-	cfg = tmp+"/config"
+	cfg = os.path.join(tmp,"config")
 	if use_stock and STOCK:
 		# copy config (sub-directories as alias):
 		def ig_dirs(dir, files):
 			return [f for f in files if os.path.isdir(os.path.join(dir, f))]
 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
-		os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d")
-		os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d")
+		os.symlink(os.path.join(STOCK_CONF_DIR,"action.d"), os.path.join(cfg,"action.d"))
+		os.symlink(os.path.join(STOCK_CONF_DIR,"filter.d"), os.path.join(cfg,"filter.d"))
 		# replace fail2ban params (database with memory):
 		r = re.compile(r'^dbfile\s*=')
-		for line in fileinput.input(cfg+"/fail2ban.conf", inplace=True):
+		for line in fileinput.input(os.path.join(cfg,"fail2ban.conf"), inplace=True):
 			line = line.rstrip('\n')
 			if r.match(line):
 				line = "dbfile = :memory:"
 			print(line)
 		# replace jail params (polling as backend to be fast in initialize):
 		r = re.compile(r'^backend\s*=')
-		for line in fileinput.input(cfg+"/jail.conf", inplace=True):
+		for line in fileinput.input(os.path.join(cfg,"jail.conf"), inplace=True):
 			line = line.rstrip('\n')
 			if r.match(line):
 				line = "backend = polling"
@@ -136,21 +136,21 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	else:
 		# just empty config directory without anything (only fail2ban.conf/jail.conf):
 		os.mkdir(cfg)
-		f = open(cfg+"/fail2ban.conf", "w")
+		f = open(os.path.join(cfg,"fail2ban.conf"), "w")
 		f.write('\n'.join((
 			"[Definition]",
 			"loglevel = INFO",
 			"logtarget = " + logtarget,
 			"syslogsocket = auto",
-			"socket = "+tmp+"/f2b.sock",
-			"pidfile = "+tmp+"/f2b.pid",
+			"socket = "+os.path.join(tmp,"f2b.sock"),
+			"pidfile = "+os.path.join(tmp,"f2b.pid"),
 			"backend = polling",
 			"dbfile = :memory:",
 			"dbpurgeage = 1d",
 			"",
 		)))
 		f.close()
-		f = open(cfg+"/jail.conf", "w")
+		f = open(os.path.join(cfg,"jail.conf"), "w")
 		f.write('\n'.join((
 			"[INCLUDES]", "",
 			"[DEFAULT]", "",
@@ -158,10 +158,10 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 		)))
 		f.close()
 		if logSys.level < logging.DEBUG: # if HEAVYDEBUG
-			_out_file(cfg+"/fail2ban.conf")
-			_out_file(cfg+"/jail.conf")
+			_out_file(os.path.join(cfg,"fail2ban.conf"))
+			_out_file(os.path.join(cfg,"jail.conf"))
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
-	return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid",
+	return ("-c", cfg, "-s", os.path.join(tmp,"f2b.sock"), "-p", os.path.join(tmp,"f2b.pid"),
 					"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
 					"--timeout", str(fail2bancmdline.MAX_WAITTIME),
 	)
@@ -234,7 +234,7 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 
 	def _wait_for_srv(self, tmp, ready=True, startparams=None):
 		try:
-			sock = tmp+"/f2b.sock"
+			sock = os.path.join(tmp,"f2b.sock")
 			# wait for server (socket):
 			ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME)
 			if not ret:
@@ -245,7 +245,7 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				if not ret:
 					raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,))
 		except: # pragma: no cover
-			log = tmp+"/f2b.log"
+			log = os.path.join(tmp,"f2b.log")
 			if os.path.isfile(log):
 				_out_file(log)
 			else:
@@ -319,7 +319,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	@with_kill_srv
 	def testClientStartBackgroundCall(self, tmp):
 		global INTERACT
-		startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
+		startparams = _start_params(tmp, logtarget=os.path.join(tmp,"f2b.log"))
 		# start (in new process, using the same python version):
 		cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
 		logSys.debug('Start %s ...', cmd)
@@ -443,29 +443,29 @@ class Fail2banClientTest(Fail2banClientServerBase):
 
 		## wrong config directory
 		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", tmp+"/miss", "start",))
-		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+			(CLIENT, "--async", "-c", os.path.join(tmp,"miss"), "start",))
+		self.assertLogged("Base configuration directory " + os.path.join(tmp,"miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
 		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",))
-		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			(CLIENT, "--async", "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"miss/f2b.sock"), "start",))
+		self.assertLogged("There is no directory " + os.path.join(tmp,"miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## not running
 		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",))
+			(CLIENT, "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"), "reload",))
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		## already exists:
-		open(tmp+"/f2b.sock", 'a').close()
+		open(os.path.join(tmp,"f2b.sock"), 'a').close()
 		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",))
+			(CLIENT, "--async", "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"), "start",))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
-		os.remove(tmp+"/f2b.sock")
+		os.remove(os.path.join(tmp,"f2b.sock"))
 
 		## wrong option:
 		self.assertRaises(FailExitException, _exec_client, 
@@ -497,7 +497,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 	@with_kill_srv
 	def testServerStartBackground(self, tmp):
 		# to prevent fork of test-cases process, start server in background via command:
-		startparams = _start_params(tmp, logtarget=tmp+"/f2b.log")
+		startparams = _start_params(tmp, logtarget=os.path.join(tmp,"f2b.log"))
 		# start (in new process, using the same python version):
 		cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
 		logSys.debug('Start %s ...', cmd)
@@ -579,20 +579,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		## wrong config directory
 		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", tmp+"/miss",))
-		self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist")
+			(SERVER, "-c", os.path.join(tmp,"miss"),))
+		self.assertLogged("Base configuration directory " + os.path.join(tmp,"miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
 		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",))
-		self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file")
+			(SERVER, "-c", os.path.join(tmp,"config"), "-x", "-s", os.path.join(tmp,"miss/f2b.sock"),))
+		self.assertLogged("There is no directory " + os.path.join(tmp,"miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## already exists:
-		open(tmp+"/f2b.sock", 'a').close()
+		open(os.path.join(tmp,"f2b.sock"), 'a').close()
 		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",))
+			(SERVER, "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"),))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
-		os.remove(tmp+"/f2b.sock")
+		os.remove(os.path.join(tmp,"f2b.sock"))

From 94cada0c6b7dd6ea4bc315c961380ded49ba9749 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 11 Jul 2016 22:42:48 -0400
Subject: [PATCH 120/623] RF(PEP8 etc): Make new fail2banclienttestcase a bit
 more readable and PEP8 friendly

---
 fail2ban/tests/fail2banclienttestcase.py | 267 +++++++++++++----------
 1 file changed, 151 insertions(+), 116 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index ed961033..69155c93 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -31,25 +31,29 @@ import time
 import signal
 import unittest
 
+from os.path import join as pjoin, isdir, isfile, exists, dirname
 from functools import wraps
 from threading import Thread
 
-
 from ..client import fail2banclient, fail2banserver, fail2bancmdline
-from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait
+from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait
 from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
 from .. import protocol
 from ..server import server
 from ..server.utils import Utils
-from .utils import LogCaptureTestCase, logSys, with_tmpdir, shutil, logging
+from .utils import LogCaptureTestCase, with_tmpdir, shutil, logging
 
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger(__name__)
 
 STOCK_CONF_DIR = "config"
-STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf'))
+STOCK = exists(pjoin(STOCK_CONF_DIR, 'fail2ban.conf'))
 
 CLIENT = "fail2ban-client"
 SERVER = "fail2ban-server"
-BIN = os.path.dirname(Fail2banServer.getServerPath())
+BIN = dirname(Fail2banServer.getServerPath())
 
 MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
 
@@ -57,7 +61,7 @@ MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
 # Several wrappers and settings for proper testing:
 #
 
-fail2bancmdline.MAX_WAITTIME = MAX_WAITTIME-1
+fail2bancmdline.MAX_WAITTIME = MAX_WAITTIME - 1
 
 fail2bancmdline.logSys = \
 fail2banclient.logSys = \
@@ -72,23 +76,43 @@ fail2banclient.output = \
 fail2banserver.output = \
 protocol.output = _test_output
 
+
+#
+# Mocking .exit so we could test its correct operation.
+# Two custom exceptions will be assessed to be raised in the tests
+#
+
+class ExitException(fail2bancmdline.ExitException):
+	"""Exception upon a normal exit"""
+	pass
+
+
+class FailExitException(fail2bancmdline.ExitException):
+	"""Exception upon abnormal exit"""
+	pass
+
+
 def _test_exit(code=0):
 	logSys.debug("Exit with code %s", code)
 	if code == 0:
 		raise ExitException()
 	else:
-		raise FailExitException()   
+		raise FailExitException()
+
 fail2bancmdline.exit = \
 fail2banclient.exit = \
 fail2banserver.exit = _test_exit
 
 INTERACT = []
+
+
 def _test_input_command(*args):
 	if len(INTERACT):
-		#logSys.debug('--- interact command: %r', INTERACT[0])
+		#logSys.debug('interact command: %r', INTERACT[0])
 		return INTERACT.pop(0)
 	else:
-		return "exit" 
+		return "exit"
+
 fail2banclient.input_command = _test_input_command
 
 # prevents change logging params, log capturing, etc:
@@ -97,38 +121,35 @@ fail2banclient.PRODUCTION = \
 fail2banserver.PRODUCTION = False
 
 
-class ExitException(fail2bancmdline.ExitException):
-	pass
-class FailExitException(fail2bancmdline.ExitException):
-	pass
-
-
-def _out_file(fn): # pragma: no cover
+def _out_file(fn):
+	"""Helper which outputs content of the file at HEAVYDEBUG loglevels"""
 	logSys.debug('---- ' + fn + ' ----')
 	for line in fileinput.input(fn):
 		line = line.rstrip('\n')
 		logSys.debug(line)
 	logSys.debug('-'*30)
 
+
 def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
-	cfg = os.path.join(tmp,"config")
+	cfg = pjoin(tmp, "config")
 	if use_stock and STOCK:
 		# copy config (sub-directories as alias):
 		def ig_dirs(dir, files):
-			return [f for f in files if os.path.isdir(os.path.join(dir, f))]
+			"""Filters list of 'files' to contain only directories (under dir)"""
+			return [f for f in files if isdir(pjoin(dir, f))]
 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
-		os.symlink(os.path.join(STOCK_CONF_DIR,"action.d"), os.path.join(cfg,"action.d"))
-		os.symlink(os.path.join(STOCK_CONF_DIR,"filter.d"), os.path.join(cfg,"filter.d"))
+		os.symlink(pjoin(STOCK_CONF_DIR, "action.d"), pjoin(cfg, "action.d"))
+		os.symlink(pjoin(STOCK_CONF_DIR, "filter.d"), pjoin(cfg, "filter.d"))
 		# replace fail2ban params (database with memory):
 		r = re.compile(r'^dbfile\s*=')
-		for line in fileinput.input(os.path.join(cfg,"fail2ban.conf"), inplace=True):
+		for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True):
 			line = line.rstrip('\n')
 			if r.match(line):
 				line = "dbfile = :memory:"
 			print(line)
 		# replace jail params (polling as backend to be fast in initialize):
 		r = re.compile(r'^backend\s*=')
-		for line in fileinput.input(os.path.join(cfg,"jail.conf"), inplace=True):
+		for line in fileinput.input(pjoin(cfg, "jail.conf"), inplace=True):
 			line = line.rstrip('\n')
 			if r.match(line):
 				line = "backend = polling"
@@ -136,64 +157,71 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	else:
 		# just empty config directory without anything (only fail2ban.conf/jail.conf):
 		os.mkdir(cfg)
-		f = open(os.path.join(cfg,"fail2ban.conf"), "w")
+		f = open(pjoin(cfg, "fail2ban.conf"), "w")
 		f.write('\n'.join((
 			"[Definition]",
 			"loglevel = INFO",
 			"logtarget = " + logtarget,
 			"syslogsocket = auto",
-			"socket = "+os.path.join(tmp,"f2b.sock"),
-			"pidfile = "+os.path.join(tmp,"f2b.pid"),
+			"socket = " + pjoin(tmp, "f2b.sock"),
+			"pidfile = " + pjoin(tmp, "f2b.pid"),
 			"backend = polling",
 			"dbfile = :memory:",
 			"dbpurgeage = 1d",
 			"",
 		)))
 		f.close()
-		f = open(os.path.join(cfg,"jail.conf"), "w")
+		f = open(pjoin(cfg, "jail.conf"), "w")
 		f.write('\n'.join((
 			"[INCLUDES]", "",
 			"[DEFAULT]", "",
 			"",
 		)))
 		f.close()
-		if logSys.level < logging.DEBUG: # if HEAVYDEBUG
-			_out_file(os.path.join(cfg,"fail2ban.conf"))
-			_out_file(os.path.join(cfg,"jail.conf"))
+		if logSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			_out_file(pjoin(cfg, "fail2ban.conf"))
+			_out_file(pjoin(cfg, "jail.conf"))
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
-	return ("-c", cfg, "-s", os.path.join(tmp,"f2b.sock"), "-p", os.path.join(tmp,"f2b.pid"),
-					"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
-					"--timeout", str(fail2bancmdline.MAX_WAITTIME),
+	return (
+		"-c", cfg, "-s", pjoin(tmp, "f2b.sock"), "-p", pjoin(tmp, "f2b.pid"),
+		"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
+		"--timeout", str(fail2bancmdline.MAX_WAITTIME),
 	)
 
-def _kill_srv(pidfile): # pragma: no cover
-	def _pid_exists(pid):
-		try:
-			os.kill(pid, 0)
-			return True
-		except OSError:
-			return False
-	logSys.debug("-- cleanup: %r", (pidfile, os.path.isdir(pidfile)))
-	if os.path.isdir(pidfile):
+
+def _pid_exists(pid):
+	"""Check if PID exists by sending 0 signal to the PID process"""
+	try:
+		os.kill(pid, 0)
+		return True
+	except OSError:
+		return False
+
+
+def _kill_srv(pidfile):
+	logSys.debug("cleanup: %r", (pidfile, isdir(pidfile)))
+	if isdir(pidfile):
 		piddir = pidfile
 		pidfile = piddir + "/f2b.pid"
-		if not os.path.isfile(pidfile):
+		if not isfile(pidfile):
 			pidfile = piddir + "/fail2ban.pid"
-	if not os.path.isfile(pidfile):
-		logSys.debug("--- cleanup: no pidfile for %r", piddir)
+
+	if not isfile(pidfile):
+		logSys.debug("cleanup: no pidfile for %r", piddir)
 		return True
+
 	f = pid = None
 	try:
-		logSys.debug("--- cleanup pidfile: %r", pidfile)
+		logSys.debug("cleanup pidfile: %r", pidfile)
 		f = open(pidfile)
 		pid = f.read().split()[1]
 		pid = int(pid)
-		logSys.debug("--- cleanup pid: %r", pid)
+		logSys.debug("cleanup pid: %r", pid)
 		if pid <= 0:
 			raise ValueError('pid %s of %s is invalid' % (pid, pidfile))
 		if not _pid_exists(pid):
 			return True
-		## try to preper stop (have signal handler):
+		## try to prepare stop (have signal handler):
 		os.kill(pid, signal.SIGTERM)
 		## check still exists after small timeout:
 		if not Utils.wait_for(lambda: not _pid_exists(pid), 1):
@@ -207,6 +235,7 @@ def _kill_srv(pidfile): # pragma: no cover
 			f.close()
 	return True
 
+
 def with_kill_srv(f):
 	"""Helper to decorate tests which receive in the last argument tmpdir to pass to kill_srv
 
@@ -234,19 +263,25 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 
 	def _wait_for_srv(self, tmp, ready=True, startparams=None):
 		try:
-			sock = os.path.join(tmp,"f2b.sock")
+			sock = pjoin(tmp, "f2b.sock")
 			# wait for server (socket):
-			ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME)
+			ret = Utils.wait_for(lambda: exists(sock), MAX_WAITTIME)
 			if not ret:
-				raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,))
+				raise Exception(
+					'Unexpected: Socket file does not exists.\nStart failed: %r'
+					% (startparams,)
+				)
 			if ready:
 				# wait for communication with worker ready:
 				ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME)
 				if not ret:
-					raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,))
-		except: # pragma: no cover
-			log = os.path.join(tmp,"f2b.log")
-			if os.path.isfile(log):
+					raise Exception(
+						'Unexpected: Server ready was not found.\nStart failed: %r'
+						% (startparams,)
+					)
+		except:  # pragma: no cover
+			log = pjoin(tmp, "f2b.log")
+			if isfile(log):
 				_out_file(log)
 			else:
 				logSys.debug("No log file %s to examine details of error", log)
@@ -256,16 +291,16 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 class Fail2banClientTest(Fail2banClientServerBase):
 
 	def testConsistency(self):
-		self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), CLIENT)))
-		self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), SERVER)))
+		self.assertTrue(isfile(pjoin(BIN, CLIENT)))
+		self.assertTrue(isfile(pjoin(BIN, SERVER)))
 
 	def testClientUsage(self):
-		self.assertRaises(ExitException, _exec_client, 
+		self.assertRaises(ExitException, _exec_client,
 			(CLIENT, "-h",))
 		self.assertLogged("Usage: " + CLIENT)
 		self.assertLogged("Report bugs to ")
 		self.pruneLog()
-		self.assertRaises(ExitException, _exec_client, 
+		self.assertRaises(ExitException, _exec_client,
 			(CLIENT, "-vq", "-V",))
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
 
@@ -273,7 +308,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	def testClientDump(self, tmp):
 		# use here the stock configuration (if possible)
 		startparams = _start_params(tmp, True)
-		self.assertRaises(ExitException, _exec_client, 
+		self.assertRaises(ExitException, _exec_client,
 			((CLIENT,) + startparams + ("-vvd",)))
 		self.assertLogged("Loading files")
 		self.assertLogged("logtarget")
@@ -284,33 +319,33 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		# use once the stock configuration (to test starting also)
 		startparams = _start_params(tmp, True)
 		# start:
-		self.assertRaises(ExitException, _exec_client, 
+		self.assertRaises(ExitException, _exec_client,
 			(CLIENT, "-b") + startparams + ("start",))
 		# wait for server (socket and ready):
 		self._wait_for_srv(tmp, True, startparams=startparams)
 		self.assertLogged("Server ready")
 		self.assertLogged("Exit with code 0")
 		try:
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_client, 
+			self.assertRaises(FailExitException, _exec_client,
 				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
 			self.pruneLog()
 			# start again (should fail):
-			self.assertRaises(FailExitException, _exec_client, 
+			self.assertRaises(FailExitException, _exec_client,
 				(CLIENT, "-b") + startparams + ("start",))
 			self.assertLogged("Server already running")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
 		self.pruneLog()
 		# stop again (should fail):
-		self.assertRaises(FailExitException, _exec_client, 
+		self.assertRaises(FailExitException, _exec_client,
 			(CLIENT,) + startparams + ("stop",))
 		self.assertLogged("Failed to access socket path")
 		self.assertLogged("Is fail2ban running?")
@@ -319,9 +354,9 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	@with_kill_srv
 	def testClientStartBackgroundCall(self, tmp):
 		global INTERACT
-		startparams = _start_params(tmp, logtarget=os.path.join(tmp,"f2b.log"))
+		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
 		# start (in new process, using the same python version):
-		cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT))
+		cmd = (sys.executable, pjoin(pjoin(BIN), CLIENT))
 		logSys.debug('Start %s ...', cmd)
 		cmd = cmd + startparams + ("--async", "start",)
 		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
@@ -332,7 +367,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.pruneLog()
 		try:
 			# echo from client (inside):
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
 			self.assertLogged("TEST-ECHO")
 			self.assertLogged("Exit with code 0")
@@ -343,7 +378,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"status",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("-i",))
 			self.assertLogged("INTERACT-ECHO")
 			self.assertLogged("Status", "Number of jail:")
@@ -355,7 +390,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"restart",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("-i",))
 			self.assertLogged("Reading config files:")
 			self.assertLogged("Shutdown successful")
@@ -367,12 +402,12 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"reload ~~unknown~jail~fail~~",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("-i",))
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.pruneLog()
 			# test reload missing jail (direct):
-			self.assertRaises(FailExitException, _exec_client, 
+			self.assertRaises(FailExitException, _exec_client,
 				(CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~"))
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.assertLogged("Exit with code -1")
@@ -380,20 +415,20 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_client, 
+			self.assertRaises(ExitException, _exec_client,
 				(CLIENT,) + startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
 	def _testClientStartForeground(self, tmp, startparams, phase):
 		# start and wait to end (foreground):
-		logSys.debug("-- start of test worker")
+		logSys.debug("start of test worker")
 		phase['start'] = True
-		self.assertRaises(fail2bancmdline.ExitException, _exec_client, 
+		self.assertRaises(fail2bancmdline.ExitException, _exec_client,
 			(CLIENT, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
-		logSys.debug("-- end of test worker")
+		logSys.debug("end of test worker")
 
 	@with_tmpdir
 	def testClientStartForeground(self, tmp):
@@ -403,7 +438,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 			startparams = _start_params(tmp, logtarget="INHERITED")
 			# because foreground block execution - start it in thread:
 			phase = dict()
-			th = Thread(name="_TestCaseWorker", 
+			th = Thread(name="_TestCaseWorker",
 				target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase))
 			th.daemon = True
 			th.start()
@@ -415,16 +450,16 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				self._wait_for_srv(tmp, True, startparams=startparams)
 				self.pruneLog()
 				# several commands to server:
-				self.assertRaises(ExitException, _exec_client, 
+				self.assertRaises(ExitException, _exec_client,
 					(CLIENT,) + startparams + ("ping",))
-				self.assertRaises(FailExitException, _exec_client, 
+				self.assertRaises(FailExitException, _exec_client,
 					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
-				self.assertRaises(ExitException, _exec_client, 
+				self.assertRaises(ExitException, _exec_client,
 					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
 			finally:
 				self.pruneLog()
 				# stop:
-				self.assertRaises(ExitException, _exec_client, 
+				self.assertRaises(ExitException, _exec_client,
 					(CLIENT,) + startparams + ("stop",))
 				# wait for end:
 				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
@@ -442,33 +477,33 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", os.path.join(tmp,"miss"), "start",))
-		self.assertLogged("Base configuration directory " + os.path.join(tmp,"miss") + " does not exist")
+		self.assertRaises(FailExitException, _exec_client,
+			(CLIENT, "--async", "-c", pjoin(tmp, "miss"), "start",))
+		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"miss/f2b.sock"), "start",))
-		self.assertLogged("There is no directory " + os.path.join(tmp,"miss") + " to contain the socket file")
+		self.assertRaises(FailExitException, _exec_client,
+			(CLIENT, "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start",))
+		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## not running
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"), "reload",))
+		self.assertRaises(FailExitException, _exec_client,
+			(CLIENT, "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload",))
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		## already exists:
-		open(os.path.join(tmp,"f2b.sock"), 'a').close()
-		self.assertRaises(FailExitException, _exec_client, 
-			(CLIENT, "--async", "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"), "start",))
+		open(pjoin(tmp, "f2b.sock"), 'a').close()
+		self.assertRaises(FailExitException, _exec_client,
+			(CLIENT, "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start",))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
-		os.remove(os.path.join(tmp,"f2b.sock"))
+		os.remove(pjoin(tmp, "f2b.sock"))
 
 		## wrong option:
-		self.assertRaises(FailExitException, _exec_client, 
+		self.assertRaises(FailExitException, _exec_client,
 			(CLIENT, "-s",))
 		self.assertLogged("Usage: ")
 		self.pruneLog()
@@ -488,7 +523,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 class Fail2banServerTest(Fail2banClientServerBase):
 
 	def testServerUsage(self):
-		self.assertRaises(ExitException, _exec_server, 
+		self.assertRaises(ExitException, _exec_server,
 			(SERVER, "-h",))
 		self.assertLogged("Usage: " + SERVER)
 		self.assertLogged("Report bugs to ")
@@ -497,9 +532,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 	@with_kill_srv
 	def testServerStartBackground(self, tmp):
 		# to prevent fork of test-cases process, start server in background via command:
-		startparams = _start_params(tmp, logtarget=os.path.join(tmp,"f2b.log"))
+		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
 		# start (in new process, using the same python version):
-		cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER))
+		cmd = (sys.executable, pjoin(pjoin(BIN), SERVER))
 		logSys.debug('Start %s ...', cmd)
 		cmd = cmd + startparams + ("-b",)
 		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
@@ -509,14 +544,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Server ready")
 		self.pruneLog()
 		try:
-			self.assertRaises(ExitException, _exec_server, 
+			self.assertRaises(ExitException, _exec_server,
 				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_server, 
+			self.assertRaises(FailExitException, _exec_server,
 				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_server, 
+			self.assertRaises(ExitException, _exec_server,
 				(SERVER,) + startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
@@ -525,7 +560,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# start and wait to end (foreground):
 		logSys.debug("-- start of test worker")
 		phase['start'] = True
-		self.assertRaises(fail2bancmdline.ExitException, _exec_server, 
+		self.assertRaises(fail2bancmdline.ExitException, _exec_server,
 			(SERVER, "-f") + startparams + ("start",))
 		# end :
 		phase['end'] = True
@@ -539,7 +574,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			startparams = _start_params(tmp, logtarget="INHERITED")
 			# because foreground block execution - start it in thread:
 			phase = dict()
-			th = Thread(name="_TestCaseWorker", 
+			th = Thread(name="_TestCaseWorker",
 				target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase))
 			th.daemon = True
 			th.start()
@@ -551,16 +586,16 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				self._wait_for_srv(tmp, True, startparams=startparams)
 				self.pruneLog()
 				# several commands to server:
-				self.assertRaises(ExitException, _exec_server, 
+				self.assertRaises(ExitException, _exec_server,
 					(SERVER,) + startparams + ("ping",))
-				self.assertRaises(FailExitException, _exec_server, 
+				self.assertRaises(FailExitException, _exec_server,
 					(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
-				self.assertRaises(ExitException, _exec_server, 
+				self.assertRaises(ExitException, _exec_server,
 					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
 			finally:
 				self.pruneLog()
 				# stop:
-				self.assertRaises(ExitException, _exec_server, 
+				self.assertRaises(ExitException, _exec_server,
 					(SERVER,) + startparams + ("stop",))
 				# wait for end:
 				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
@@ -578,21 +613,21 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", os.path.join(tmp,"miss"),))
-		self.assertLogged("Base configuration directory " + os.path.join(tmp,"miss") + " does not exist")
+		self.assertRaises(FailExitException, _exec_server,
+			(SERVER, "-c", pjoin(tmp, "miss"),))
+		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", os.path.join(tmp,"config"), "-x", "-s", os.path.join(tmp,"miss/f2b.sock"),))
-		self.assertLogged("There is no directory " + os.path.join(tmp,"miss") + " to contain the socket file")
+		self.assertRaises(FailExitException, _exec_server,
+			(SERVER, "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"),))
+		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## already exists:
-		open(os.path.join(tmp,"f2b.sock"), 'a').close()
-		self.assertRaises(FailExitException, _exec_server, 
-			(SERVER, "-c", os.path.join(tmp,"config"), "-s", os.path.join(tmp,"f2b.sock"),))
+		open(pjoin(tmp, "f2b.sock"), 'a').close()
+		self.assertRaises(FailExitException, _exec_server,
+			(SERVER, "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"),))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
-		os.remove(os.path.join(tmp,"f2b.sock"))
+		os.remove(pjoin(tmp, "f2b.sock"))

From 4df06f535294060218f968a4733df350dfa2e239 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 11 Jul 2016 22:44:47 -0400
Subject: [PATCH 121/623] ENH: testcases - verbosity > 2 now also includes
 logger name (to replace need for custom --s)

---
 bin/fail2ban-testcases | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 20d444f0..748737a5 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -122,7 +122,7 @@ if verbosity > 1: # pragma: no cover
 	if verbosity > 3:
 		fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
 	if verbosity > 2:
-		fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt
+		fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25s %(levelname)-5.5s' + fmt
 	else:
 		fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
 #

From 1417cc99ef54b7dd1da83509a23531e984223993 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 11 Jul 2016 23:26:30 -0400
Subject: [PATCH 122/623] RF: group common tests in base class, specialize
 assesment of good/bad runs per client/server

---
 fail2ban/tests/fail2banclienttestcase.py | 270 ++++++++++-------------
 1 file changed, 116 insertions(+), 154 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 69155c93..e69d31e1 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -287,29 +287,91 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				logSys.debug("No log file %s to examine details of error", log)
 			raise
 
+	def assertExitsNormally(self, args):
+		raise NotImplementedError("To be defined in subclass")
+
+	def assertExitsAbnormally(self, args):
+		raise NotImplementedError("To be defined in subclass")
+
+	#
+	# Common tests
+	#
+	def _testStartForeground(self, tmp, startparams, phase):
+		# start and wait to end (foreground):
+		logSys.debug("start of test worker")
+		phase['start'] = True
+		self.assertExitsNormally(("-f",) + startparams + ("start",))
+		# end :
+		phase['end'] = True
+		logSys.debug("end of test worker")
+
+	@with_tmpdir
+	def testStartForeground(self, tmp):
+		# intended to be ran only in subclasses
+		th = None
+		try:
+			# started directly here, so prevent overwrite test cases logger with "INHERITED"
+			startparams = _start_params(tmp, logtarget="INHERITED")
+			# because foreground block execution - start it in thread:
+			phase = dict()
+			th = Thread(
+				name="_TestCaseWorker",
+				target=self._testStartForeground,
+				args=(tmp, startparams, phase)
+			)
+			th.daemon = True
+			th.start()
+			try:
+				# wait for start thread:
+				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('start', None))
+				# wait for server (socket and ready):
+				self._wait_for_srv(tmp, True, startparams=startparams)
+				self.pruneLog()
+				# several commands to server:
+				self.assertExitsNormally(startparams + ("ping",))
+				self.assertExitsAbnormally(startparams + ("~~unknown~cmd~failed~~",))
+				self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
+			finally:
+				self.pruneLog()
+				# stop:
+				self.assertExitsNormally(startparams + ("stop",))
+				# wait for end:
+				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('end', None))
+				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+		finally:
+			_kill_srv(tmp)
+			if th:
+				th.join()
+
 
 class Fail2banClientTest(Fail2banClientServerBase):
 
+	def assertExitsNormally(self, args):
+		self.assertRaises(ExitException, _exec_client, ((CLIENT,) + args))
+
+	def assertExitsAbnormally(self, args):
+		self.assertRaises(FailExitException, _exec_client, ((CLIENT,) + args))
+
 	def testConsistency(self):
 		self.assertTrue(isfile(pjoin(BIN, CLIENT)))
 		self.assertTrue(isfile(pjoin(BIN, SERVER)))
 
 	def testClientUsage(self):
-		self.assertRaises(ExitException, _exec_client,
-			(CLIENT, "-h",))
+		self.assertExitsNormally(("-h",))
 		self.assertLogged("Usage: " + CLIENT)
 		self.assertLogged("Report bugs to ")
 		self.pruneLog()
-		self.assertRaises(ExitException, _exec_client,
-			(CLIENT, "-vq", "-V",))
+		self.assertExitsNormally(("-vq", "-V",))
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
 
 	@with_tmpdir
 	def testClientDump(self, tmp):
 		# use here the stock configuration (if possible)
 		startparams = _start_params(tmp, True)
-		self.assertRaises(ExitException, _exec_client,
-			((CLIENT,) + startparams + ("-vvd",)))
+		self.assertExitsNormally(
+			(startparams + ("-vvd",)))
 		self.assertLogged("Loading files")
 		self.assertLogged("logtarget")
 
@@ -319,34 +381,34 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		# use once the stock configuration (to test starting also)
 		startparams = _start_params(tmp, True)
 		# start:
-		self.assertRaises(ExitException, _exec_client,
-			(CLIENT, "-b") + startparams + ("start",))
+		self.assertExitsNormally(
+			("-b",) + startparams + ("start",))
 		# wait for server (socket and ready):
 		self._wait_for_srv(tmp, True, startparams=startparams)
 		self.assertLogged("Server ready")
 		self.assertLogged("Exit with code 0")
 		try:
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_client,
-				(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
+			self.assertExitsNormally(
+				startparams + ("echo", "TEST-ECHO",))
+			self.assertExitsAbnormally(
+				startparams + ("~~unknown~cmd~failed~~",))
 			self.pruneLog()
 			# start again (should fail):
-			self.assertRaises(FailExitException, _exec_client,
-				(CLIENT, "-b") + startparams + ("start",))
+			self.assertExitsAbnormally(
+				("-b",) + startparams + ("start",))
 			self.assertLogged("Server already running")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("stop",))
+			self.assertExitsNormally(
+				startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
 		self.pruneLog()
 		# stop again (should fail):
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT,) + startparams + ("stop",))
+		self.assertExitsAbnormally(
+			startparams + ("stop",))
 		self.assertLogged("Failed to access socket path")
 		self.assertLogged("Is fail2ban running?")
 
@@ -367,8 +429,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.pruneLog()
 		try:
 			# echo from client (inside):
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
+			self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
 			self.assertLogged("TEST-ECHO")
 			self.assertLogged("Exit with code 0")
 			self.pruneLog()
@@ -378,8 +439,8 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"status",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("-i",))
+			self.assertExitsNormally(
+				startparams + ("-i",))
 			self.assertLogged("INTERACT-ECHO")
 			self.assertLogged("Status", "Number of jail:")
 			self.assertLogged("Exit with code 0")
@@ -390,8 +451,8 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"restart",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("-i",))
+			self.assertExitsNormally(
+				startparams + ("-i",))
 			self.assertLogged("Reading config files:")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Server ready")
@@ -402,74 +463,23 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"reload ~~unknown~jail~fail~~",
 				"exit"
 			]
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("-i",))
+			self.assertExitsNormally(
+				startparams + ("-i",))
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.pruneLog()
 			# test reload missing jail (direct):
-			self.assertRaises(FailExitException, _exec_client,
-				(CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~"))
+			self.assertExitsAbnormally(
+				startparams + ("reload", "~~unknown~jail~fail~~"))
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.assertLogged("Exit with code -1")
 			self.pruneLog()
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_client,
-				(CLIENT,) + startparams + ("stop",))
+			self.assertExitsNormally(startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
-	def _testClientStartForeground(self, tmp, startparams, phase):
-		# start and wait to end (foreground):
-		logSys.debug("start of test worker")
-		phase['start'] = True
-		self.assertRaises(fail2bancmdline.ExitException, _exec_client,
-			(CLIENT, "-f") + startparams + ("start",))
-		# end :
-		phase['end'] = True
-		logSys.debug("end of test worker")
-
-	@with_tmpdir
-	def testClientStartForeground(self, tmp):
-		th = None
-		try:
-			# started directly here, so prevent overwrite test cases logger with "INHERITED"
-			startparams = _start_params(tmp, logtarget="INHERITED")
-			# because foreground block execution - start it in thread:
-			phase = dict()
-			th = Thread(name="_TestCaseWorker",
-				target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase))
-			th.daemon = True
-			th.start()
-			try:
-				# wait for start thread:
-				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('start', None))
-				# wait for server (socket and ready):
-				self._wait_for_srv(tmp, True, startparams=startparams)
-				self.pruneLog()
-				# several commands to server:
-				self.assertRaises(ExitException, _exec_client,
-					(CLIENT,) + startparams + ("ping",))
-				self.assertRaises(FailExitException, _exec_client,
-					(CLIENT,) + startparams + ("~~unknown~cmd~failed~~",))
-				self.assertRaises(ExitException, _exec_client,
-					(CLIENT,) + startparams + ("echo", "TEST-ECHO",))
-			finally:
-				self.pruneLog()
-				# stop:
-				self.assertRaises(ExitException, _exec_client,
-					(CLIENT,) + startparams + ("stop",))
-				# wait for end:
-				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('end', None))
-				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-		finally:
-			_kill_srv(tmp)
-			if th:
-				th.join()
-
 	@with_tmpdir
 	@with_kill_srv
 	def testClientFailStart(self, tmp):
@@ -477,34 +487,34 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT, "--async", "-c", pjoin(tmp, "miss"), "start",))
+		self.assertExitsAbnormally(
+			("--async", "-c", pjoin(tmp, "miss"), "start",))
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT, "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start",))
+		self.assertExitsAbnormally(
+			("--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start",))
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## not running
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT, "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload",))
+		self.assertExitsAbnormally(
+			("-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload",))
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT, "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start",))
+		self.assertExitsAbnormally(
+			("--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start",))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))
 
 		## wrong option:
-		self.assertRaises(FailExitException, _exec_client,
-			(CLIENT, "-s",))
+		self.assertExitsAbnormally(
+			("-s",))
 		self.assertLogged("Usage: ")
 		self.pruneLog()
 
@@ -522,9 +532,14 @@ class Fail2banClientTest(Fail2banClientServerBase):
 
 class Fail2banServerTest(Fail2banClientServerBase):
 
+	def assertExitsNormally(self, args):
+		self.assertRaises(ExitException, _exec_server, ((SERVER,) + args))
+
+	def assertExitsAbnormally(self, args):
+		self.assertRaises(FailExitException, _exec_server, ((SERVER,) + args))
+
 	def testServerUsage(self):
-		self.assertRaises(ExitException, _exec_server,
-			(SERVER, "-h",))
+		self.assertExitsNormally(("-h",))
 		self.assertLogged("Usage: " + SERVER)
 		self.assertLogged("Report bugs to ")
 
@@ -544,68 +559,15 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Server ready")
 		self.pruneLog()
 		try:
-			self.assertRaises(ExitException, _exec_server,
-				(SERVER,) + startparams + ("echo", "TEST-ECHO",))
-			self.assertRaises(FailExitException, _exec_server,
-				(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
+			self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
+			self.assertExitsAbnormally(startparams + ("~~unknown~cmd~failed~~",))
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertRaises(ExitException, _exec_server,
-				(SERVER,) + startparams + ("stop",))
+			self.assertExitsNormally(startparams + ("stop",))
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
-	def _testServerStartForeground(self, tmp, startparams, phase):
-		# start and wait to end (foreground):
-		logSys.debug("-- start of test worker")
-		phase['start'] = True
-		self.assertRaises(fail2bancmdline.ExitException, _exec_server,
-			(SERVER, "-f") + startparams + ("start",))
-		# end :
-		phase['end'] = True
-		logSys.debug("-- end of test worker")
-
-	@with_tmpdir
-	def testServerStartForeground(self, tmp):
-		th = None
-		try:
-			# started directly here, so prevent overwrite test cases logger with "INHERITED"
-			startparams = _start_params(tmp, logtarget="INHERITED")
-			# because foreground block execution - start it in thread:
-			phase = dict()
-			th = Thread(name="_TestCaseWorker",
-				target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase))
-			th.daemon = True
-			th.start()
-			try:
-				# wait for start thread:
-				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('start', None))
-				# wait for server (socket and ready):
-				self._wait_for_srv(tmp, True, startparams=startparams)
-				self.pruneLog()
-				# several commands to server:
-				self.assertRaises(ExitException, _exec_server,
-					(SERVER,) + startparams + ("ping",))
-				self.assertRaises(FailExitException, _exec_server,
-					(SERVER,) + startparams + ("~~unknown~cmd~failed~~",))
-				self.assertRaises(ExitException, _exec_server,
-					(SERVER,) + startparams + ("echo", "TEST-ECHO",))
-			finally:
-				self.pruneLog()
-				# stop:
-				self.assertRaises(ExitException, _exec_server,
-					(SERVER,) + startparams + ("stop",))
-				# wait for end:
-				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('end', None))
-				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-		finally:
-			_kill_srv(tmp)
-			if th:
-				th.join()
-
 	@with_tmpdir
 	@with_kill_srv
 	def testServerFailStart(self, tmp):
@@ -613,21 +575,21 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertRaises(FailExitException, _exec_server,
-			(SERVER, "-c", pjoin(tmp, "miss"),))
+		self.assertExitsAbnormally(
+			("-c", pjoin(tmp, "miss"),))
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertRaises(FailExitException, _exec_server,
-			(SERVER, "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"),))
+		self.assertExitsAbnormally(
+			("-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"),))
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.assertRaises(FailExitException, _exec_server,
-			(SERVER, "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"),))
+		self.assertExitsAbnormally(
+			("-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"),))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))

From fcda7c9ac7b3e8585314bbdb4d6615ed0d2a0ece Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 11 Jul 2016 23:54:52 -0400
Subject: [PATCH 123/623] RF: revertably mock out exit call while testing new
 client/servers

---
 fail2ban/client/fail2bancmdline.py       | 14 ++++++++++++--
 fail2ban/tests/fail2banclienttestcase.py | 23 ++++++++++++-----------
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 9110a2b8..19f596a2 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -42,6 +42,7 @@ PRODUCTION = True
 
 MAX_WAITTIME = 30
 
+
 class Fail2banCmdLine():
 
 	def __init__(self):
@@ -256,14 +257,23 @@ class Fail2banCmdLine():
 			output(c)
 		return True
 
+	#
+	# _exit is made to ease mocking out of the behaviour in tests,
+	# since method is also exposed in API via globally bound variable
 	@staticmethod
-	def exit(code=0): # pragma: no cover - can't test
-		logSys.debug("Exit with code %s", code)
+	def _exit(code=0):
 		if hasattr(os, '_exit') and os._exit:
 			os._exit(code)
 		else:
 			sys.exit(code)
 
+	@staticmethod
+	def exit(code=0):
+		logSys.debug("Exit with code %s", code)
+		# import pdb; pdb.set_trace()
+		Fail2banCmdLine._exit(code)
+
+
 # global exit handler:
 exit = Fail2banCmdLine.exit
 
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e69d31e1..fa8a210c 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -36,6 +36,7 @@ from functools import wraps
 from threading import Thread
 
 from ..client import fail2banclient, fail2banserver, fail2bancmdline
+from ..client.fail2bancmdline import Fail2banCmdLine
 from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait
 from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
 from .. import protocol
@@ -92,17 +93,6 @@ class FailExitException(fail2bancmdline.ExitException):
 	pass
 
 
-def _test_exit(code=0):
-	logSys.debug("Exit with code %s", code)
-	if code == 0:
-		raise ExitException()
-	else:
-		raise FailExitException()
-
-fail2bancmdline.exit = \
-fail2banclient.exit = \
-fail2banserver.exit = _test_exit
-
 INTERACT = []
 
 
@@ -256,11 +246,22 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		LogCaptureTestCase.setUp(self)
+		Fail2banCmdLine._exit = staticmethod(self._test_exit)
 
 	def tearDown(self):
 		"""Call after every test case."""
+		Fail2banCmdLine._exit = self._orig_exit
 		LogCaptureTestCase.tearDown(self)
 
+	_orig_exit = Fail2banCmdLine._exit
+
+	@staticmethod
+	def _test_exit(code=0):
+		if code == 0:
+			raise ExitException()
+		else:
+			raise FailExitException()
+
 	def _wait_for_srv(self, tmp, ready=True, startparams=None):
 		try:
 			sock = pjoin(tmp, "f2b.sock")

From 3c42b4f56e5e20a56e22d534371ff7adbc2ce78b Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 12 Jul 2016 00:06:41 -0400
Subject: [PATCH 124/623] minor residual cleanups

---
 fail2ban/client/fail2bancmdline.py       | 1 -
 fail2ban/tests/fail2banclienttestcase.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 19f596a2..aadf823a 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -270,7 +270,6 @@ class Fail2banCmdLine():
 	@staticmethod
 	def exit(code=0):
 		logSys.debug("Exit with code %s", code)
-		# import pdb; pdb.set_trace()
 		Fail2banCmdLine._exit(code)
 
 
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index fa8a210c..24b396f5 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -419,7 +419,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		global INTERACT
 		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
 		# start (in new process, using the same python version):
-		cmd = (sys.executable, pjoin(pjoin(BIN), CLIENT))
+		cmd = (sys.executable, pjoin(BIN, CLIENT))
 		logSys.debug('Start %s ...', cmd)
 		cmd = cmd + startparams + ("--async", "start",)
 		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
@@ -550,7 +550,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# to prevent fork of test-cases process, start server in background via command:
 		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
 		# start (in new process, using the same python version):
-		cmd = (sys.executable, pjoin(pjoin(BIN), SERVER))
+		cmd = (sys.executable, pjoin(BIN, SERVER))
 		logSys.debug('Start %s ...', cmd)
 		cmd = cmd + startparams + ("-b",)
 		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)

From 7242c9cbdeb9dea90a5d6068d881823fda9d6139 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 12 Jul 2016 12:02:05 +0200
Subject: [PATCH 125/623] code review after enhancements of Yaroslav

---
 bin/fail2ban-testcases                   |   4 +-
 fail2ban/tests/fail2banclienttestcase.py | 124 +++++++++++------------
 2 files changed, 60 insertions(+), 68 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 748737a5..98b9118f 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -30,7 +30,7 @@ import sys
 import time
 import unittest
 
-# Check if local fail2ban module exists, and use if it exists by 
+# Check if local fail2ban module exists, and use if it exists by
 # modifying the path. This is such that tests can be used in dev
 # environment.
 if os.path.exists("fail2ban/__init__.py"):
@@ -122,7 +122,7 @@ if verbosity > 1: # pragma: no cover
 	if verbosity > 3:
 		fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
 	if verbosity > 2:
-		fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25s %(levelname)-5.5s' + fmt
+		fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
 	else:
 		fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
 #
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 24b396f5..e0ec8dcd 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -192,9 +192,9 @@ def _kill_srv(pidfile):
 	logSys.debug("cleanup: %r", (pidfile, isdir(pidfile)))
 	if isdir(pidfile):
 		piddir = pidfile
-		pidfile = piddir + "/f2b.pid"
-		if not isfile(pidfile):
-			pidfile = piddir + "/fail2ban.pid"
+		pidfile = pjoin(piddir, "f2b.pid")
+		if not isfile(pidfile): # pragma: no cover
+			pidfile = pjoin(piddir, "fail2ban.pid")
 
 	if not isfile(pidfile):
 		logSys.debug("cleanup: no pidfile for %r", piddir)
@@ -243,6 +243,8 @@ def with_kill_srv(f):
 
 class Fail2banClientServerBase(LogCaptureTestCase):
 
+	_orig_exit = Fail2banCmdLine._exit
+
 	def setUp(self):
 		"""Call before every test case."""
 		LogCaptureTestCase.setUp(self)
@@ -253,8 +255,6 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		Fail2banCmdLine._exit = self._orig_exit
 		LogCaptureTestCase.tearDown(self)
 
-	_orig_exit = Fail2banCmdLine._exit
-
 	@staticmethod
 	def _test_exit(code=0):
 		if code == 0:
@@ -288,10 +288,10 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				logSys.debug("No log file %s to examine details of error", log)
 			raise
 
-	def assertExitsNormally(self, args):
+	def execSuccess(self, startparams, *args):
 		raise NotImplementedError("To be defined in subclass")
 
-	def assertExitsAbnormally(self, args):
+	def execFailed(self, startparams, *args):
 		raise NotImplementedError("To be defined in subclass")
 
 	#
@@ -301,7 +301,7 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		# start and wait to end (foreground):
 		logSys.debug("start of test worker")
 		phase['start'] = True
-		self.assertExitsNormally(("-f",) + startparams + ("start",))
+		self.execSuccess(("-f",) + startparams, "start")
 		# end :
 		phase['end'] = True
 		logSys.debug("end of test worker")
@@ -330,13 +330,13 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				self._wait_for_srv(tmp, True, startparams=startparams)
 				self.pruneLog()
 				# several commands to server:
-				self.assertExitsNormally(startparams + ("ping",))
-				self.assertExitsAbnormally(startparams + ("~~unknown~cmd~failed~~",))
-				self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
+				self.execSuccess(startparams, "ping")
+				self.execFailed(startparams, "~~unknown~cmd~failed~~")
+				self.execSuccess(startparams, "echo", "TEST-ECHO")
 			finally:
 				self.pruneLog()
 				# stop:
-				self.assertExitsNormally(startparams + ("stop",))
+				self.execSuccess(startparams, "stop")
 				# wait for end:
 				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
 				self.assertTrue(phase.get('end', None))
@@ -349,30 +349,31 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 
 class Fail2banClientTest(Fail2banClientServerBase):
 
-	def assertExitsNormally(self, args):
-		self.assertRaises(ExitException, _exec_client, ((CLIENT,) + args))
+	def execSuccess(self, startparams, *args):
+		self.assertRaises(ExitException, _exec_client,
+			((CLIENT,) + startparams + args))
 
-	def assertExitsAbnormally(self, args):
-		self.assertRaises(FailExitException, _exec_client, ((CLIENT,) + args))
+	def execFailed(self, startparams, *args):
+		self.assertRaises(FailExitException, _exec_client,
+			((CLIENT,) + startparams + args))
 
 	def testConsistency(self):
 		self.assertTrue(isfile(pjoin(BIN, CLIENT)))
 		self.assertTrue(isfile(pjoin(BIN, SERVER)))
 
 	def testClientUsage(self):
-		self.assertExitsNormally(("-h",))
+		self.execSuccess((), "-h")
 		self.assertLogged("Usage: " + CLIENT)
 		self.assertLogged("Report bugs to ")
 		self.pruneLog()
-		self.assertExitsNormally(("-vq", "-V",))
+		self.execSuccess((), "-vq", "-V")
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
 
 	@with_tmpdir
 	def testClientDump(self, tmp):
 		# use here the stock configuration (if possible)
 		startparams = _start_params(tmp, True)
-		self.assertExitsNormally(
-			(startparams + ("-vvd",)))
+		self.execSuccess(startparams, "-vvd")
 		self.assertLogged("Loading files")
 		self.assertLogged("logtarget")
 
@@ -382,34 +383,28 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		# use once the stock configuration (to test starting also)
 		startparams = _start_params(tmp, True)
 		# start:
-		self.assertExitsNormally(
-			("-b",) + startparams + ("start",))
+		self.execSuccess(("-b",) + startparams, "start")
 		# wait for server (socket and ready):
 		self._wait_for_srv(tmp, True, startparams=startparams)
 		self.assertLogged("Server ready")
 		self.assertLogged("Exit with code 0")
 		try:
-			self.assertExitsNormally(
-				startparams + ("echo", "TEST-ECHO",))
-			self.assertExitsAbnormally(
-				startparams + ("~~unknown~cmd~failed~~",))
+			self.execSuccess(startparams, "echo", "TEST-ECHO")
+			self.execFailed(startparams, "~~unknown~cmd~failed~~")
 			self.pruneLog()
 			# start again (should fail):
-			self.assertExitsAbnormally(
-				("-b",) + startparams + ("start",))
+			self.execFailed(("-b",) + startparams, "start")
 			self.assertLogged("Server already running")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertExitsNormally(
-				startparams + ("stop",))
+			self.execSuccess(startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
 		self.pruneLog()
 		# stop again (should fail):
-		self.assertExitsAbnormally(
-			startparams + ("stop",))
+		self.execFailed(startparams, "stop")
 		self.assertLogged("Failed to access socket path")
 		self.assertLogged("Is fail2ban running?")
 
@@ -430,7 +425,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.pruneLog()
 		try:
 			# echo from client (inside):
-			self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
+			self.execSuccess(startparams, "echo", "TEST-ECHO")
 			self.assertLogged("TEST-ECHO")
 			self.assertLogged("Exit with code 0")
 			self.pruneLog()
@@ -440,8 +435,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"status",
 				"exit"
 			]
-			self.assertExitsNormally(
-				startparams + ("-i",))
+			self.execSuccess(startparams, "-i")
 			self.assertLogged("INTERACT-ECHO")
 			self.assertLogged("Status", "Number of jail:")
 			self.assertLogged("Exit with code 0")
@@ -452,8 +446,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"restart",
 				"exit"
 			]
-			self.assertExitsNormally(
-				startparams + ("-i",))
+			self.execSuccess(startparams, "-i")
 			self.assertLogged("Reading config files:")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Server ready")
@@ -464,20 +457,18 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"reload ~~unknown~jail~fail~~",
 				"exit"
 			]
-			self.assertExitsNormally(
-				startparams + ("-i",))
+			self.execSuccess(startparams, "-i")
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.pruneLog()
 			# test reload missing jail (direct):
-			self.assertExitsAbnormally(
-				startparams + ("reload", "~~unknown~jail~fail~~"))
+			self.execFailed(startparams, "reload", "~~unknown~jail~fail~~")
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.assertLogged("Exit with code -1")
 			self.pruneLog()
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertExitsNormally(startparams + ("stop",))
+			self.execSuccess(startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
@@ -488,34 +479,33 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertExitsAbnormally(
-			("--async", "-c", pjoin(tmp, "miss"), "start",))
+		self.execFailed((),
+			"--async", "-c", pjoin(tmp, "miss"), "start")
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertExitsAbnormally(
-			("--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start",))
+		self.execFailed((),
+			"--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start")
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## not running
-		self.assertExitsAbnormally(
-			("-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload",))
+		self.execFailed((),
+			"-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload")
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.assertExitsAbnormally(
-			("--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start",))
+		self.execFailed((),
+			"--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start")
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))
 
 		## wrong option:
-		self.assertExitsAbnormally(
-			("-s",))
+		self.execFailed((), "-s")
 		self.assertLogged("Usage: ")
 		self.pruneLog()
 
@@ -533,14 +523,16 @@ class Fail2banClientTest(Fail2banClientServerBase):
 
 class Fail2banServerTest(Fail2banClientServerBase):
 
-	def assertExitsNormally(self, args):
-		self.assertRaises(ExitException, _exec_server, ((SERVER,) + args))
+	def execSuccess(self, startparams, *args):
+		self.assertRaises(ExitException, _exec_server,
+			((SERVER,) + startparams + args))
 
-	def assertExitsAbnormally(self, args):
-		self.assertRaises(FailExitException, _exec_server, ((SERVER,) + args))
+	def execFailed(self, startparams, *args):
+		self.assertRaises(FailExitException, _exec_server,
+			((SERVER,) + startparams + args))
 
 	def testServerUsage(self):
-		self.assertExitsNormally(("-h",))
+		self.execSuccess((), "-h")
 		self.assertLogged("Usage: " + SERVER)
 		self.assertLogged("Report bugs to ")
 
@@ -560,12 +552,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Server ready")
 		self.pruneLog()
 		try:
-			self.assertExitsNormally(startparams + ("echo", "TEST-ECHO",))
-			self.assertExitsAbnormally(startparams + ("~~unknown~cmd~failed~~",))
+			self.execSuccess(startparams, "echo", "TEST-ECHO")
+			self.execFailed(startparams, "~~unknown~cmd~failed~~")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.assertExitsNormally(startparams + ("stop",))
+			self.execSuccess(startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
@@ -576,21 +568,21 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.assertExitsAbnormally(
-			("-c", pjoin(tmp, "miss"),))
+		self.execFailed((),
+			"-c", pjoin(tmp, "miss"))
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.assertExitsAbnormally(
-			("-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"),))
+		self.execFailed((),
+			"-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"))
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.assertExitsAbnormally(
-			("-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"),))
+		self.execFailed((),
+			"-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))

From 0dc3db1f43dc73de91c46556c7b6083cfd42e99c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 12 Jul 2016 12:06:48 +0200
Subject: [PATCH 126/623] test of `_kill_srv` added, don't kill server if
 foreground; pid_exists already available in Utils; small bug and stability
 fixing;

---
 fail2ban/tests/fail2banclienttestcase.py | 81 ++++++++++++++++--------
 1 file changed, 54 insertions(+), 27 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e0ec8dcd..2a18d204 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -98,7 +98,7 @@ INTERACT = []
 
 def _test_input_command(*args):
 	if len(INTERACT):
-		#logSys.debug('interact command: %r', INTERACT[0])
+		#logSys.debug('--- interact command: %r', INTERACT[0])
 		return INTERACT.pop(0)
 	else:
 		return "exit"
@@ -179,15 +179,6 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	)
 
 
-def _pid_exists(pid):
-	"""Check if PID exists by sending 0 signal to the PID process"""
-	try:
-		os.kill(pid, 0)
-		return True
-	except OSError:
-		return False
-
-
 def _kill_srv(pidfile):
 	logSys.debug("cleanup: %r", (pidfile, isdir(pidfile)))
 	if isdir(pidfile):
@@ -204,25 +195,32 @@ def _kill_srv(pidfile):
 	try:
 		logSys.debug("cleanup pidfile: %r", pidfile)
 		f = open(pidfile)
-		pid = f.read().split()[1]
+		pid = f.read()
+		pid = re.match(r'\S+', pid).group()
 		pid = int(pid)
-		logSys.debug("cleanup pid: %r", pid)
-		if pid <= 0:
-			raise ValueError('pid %s of %s is invalid' % (pid, pidfile))
-		if not _pid_exists(pid):
-			return True
-		## try to prepare stop (have signal handler):
-		os.kill(pid, signal.SIGTERM)
-		## check still exists after small timeout:
-		if not Utils.wait_for(lambda: not _pid_exists(pid), 1):
-			## try to kill hereafter:
-			os.kill(pid, signal.SIGKILL)
-		return not _pid_exists(pid)
-	except Exception as e:
+	except Exception as e: # pragma: no cover
 		logSys.debug(e)
+		return False
 	finally:
 		if f is not None:
 			f.close()
+
+	try:
+		logSys.debug("cleanup pid: %r", pid)
+		if pid <= 0 or pid == os.getpid(): # pragma: no cover
+			raise ValueError('pid %s of %s is invalid' % (pid, pidfile))
+		if not Utils.pid_exists(pid):
+			return True
+		## try to properly stop (have signal handler):
+		os.kill(pid, signal.SIGTERM)
+		## check still exists after small timeout:
+		if not Utils.wait_for(lambda: not Utils.pid_exists(pid), 1):
+			## try to kill hereafter:
+			os.kill(pid, signal.SIGKILL)
+		logSys.debug("cleanup: kill ready")
+		return not Utils.pid_exists(pid)
+	except Exception as e: # pragma: no cover
+		logSys.exception(e)
 	return True
 
 
@@ -310,11 +308,11 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 	def testStartForeground(self, tmp):
 		# intended to be ran only in subclasses
 		th = None
+		phase = dict()
 		try:
 			# started directly here, so prevent overwrite test cases logger with "INHERITED"
 			startparams = _start_params(tmp, logtarget="INHERITED")
 			# because foreground block execution - start it in thread:
-			phase = dict()
 			th = Thread(
 				name="_TestCaseWorker",
 				target=self._testStartForeground,
@@ -342,9 +340,11 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				self.assertTrue(phase.get('end', None))
 				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
 		finally:
-			_kill_srv(tmp)
 			if th:
-				th.join()
+				# we start client/server directly in current process (new thread),
+				# so don't kill (same process) - if success, just wait for end of worker:
+				if phase.get('end', None):
+					th.join()
 
 
 class Fail2banClientTest(Fail2banClientServerBase):
@@ -586,3 +586,30 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))
+
+	@with_tmpdir
+	def testKillAfterStart(self, tmp):
+		try:
+			# to prevent fork of test-cases process, start server in background via command:
+			startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
+			# start (in new process, using the same python version):
+			cmd = (sys.executable, pjoin(BIN, SERVER))
+			logSys.debug('Start %s ...', cmd)
+			cmd = cmd + startparams + ("-b",)
+			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			self.assertTrue(len(ret) and ret[0])
+			# wait for server (socket and ready):
+			self._wait_for_srv(tmp, True, startparams=cmd)
+			self.assertLogged("Server ready")
+			self.pruneLog()
+			logSys.debug('Kill server ... %s', tmp)
+		finally:
+			self.assertTrue(_kill_srv(tmp))
+		# wait for end (kill was successful):
+		Utils.wait_for(lambda: not isfile(pjoin(tmp, "f2b.pid")), MAX_WAITTIME)
+		self.assertFalse(isfile(pjoin(tmp, "f2b.pid")))
+		self.assertLogged("cleanup: kill ready")
+		self.pruneLog()
+		# again:
+		self.assertTrue(_kill_srv(tmp))
+		self.assertLogged("cleanup: no pidfile for")

From 52cd8a9a960d24c042e6181a83e44ef49cbd54b2 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 12 Jul 2016 22:24:56 -0400
Subject: [PATCH 127/623] RF: _rebindSignal helper to centralize assignment of
 new signal handlers + minor PEP8ing

---
 fail2ban/server/server.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index bade12cb..7edf19e9 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -87,6 +87,11 @@ class Server:
 		logSys.debug("Caught signal %d. Flushing logs" % signum)
 		self.flushLogs()
 
+	def _rebindSignal(self, s, new):
+		"""Bind new signal handler while storing old one in _prev_signals"""
+		self.__prev_signals[s] = signal.getsignal(s)
+		signal.signal(s, new)
+
 	def start(self, sock, pidfile, force=False, conf={}):
 		# First set the mask to only allow access to owner
 		os.umask(0077)
@@ -120,9 +125,10 @@ class Server:
 
 		# Install signal handlers
 		if _thread_name() == '_MainThread':
-			for s in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1):
-				self.__prev_signals[s] = signal.getsignal(s)
-				signal.signal(s, self.__sigTERMhandler if s != signal.SIGUSR1 else self.__sigUSR1handler)
+			for s in (signal.SIGTERM, signal.SIGINT):
+				self._rebindSignal(s, self.__sigTERMhandler)
+			self._rebindSignal(signal.SIGUSR1, self.__sigUSR1handler)
+
 		# Ensure unhandled exceptions are logged
 		sys.excepthook = excepthook
 
@@ -490,18 +496,18 @@ class Server:
 				try:
 					handler.flush()
 					handler.close()
-				except (ValueError, KeyError): # pragma: no cover
+				except (ValueError, KeyError):  # pragma: no cover
 					# Is known to be thrown after logging was shutdown once
 					# with older Pythons -- seems to be safe to ignore there
 					# At least it was still failing on 2.6.2-0ubuntu1 (jaunty)
-					if (2,6,3) <= sys.version_info < (3,) or \
-							(3,2) <= sys.version_info:
+					if (2, 6, 3) <= sys.version_info < (3,) or \
+							(3, 2) <= sys.version_info:
 						raise
 			# tell the handler to use this format
 			hdlr.setFormatter(formatter)
 			logger.addHandler(hdlr)
 			# Does not display this message at startup.
-			if not self.__logTarget is None:
+			if self.__logTarget is not None:
 				logSys.info("Start Fail2ban v%s", version.version)
 				logSys.info(
 					"Changed logging target to %s for Fail2ban v%s"
@@ -588,9 +594,7 @@ class Server:
 		# We need to set this in the parent process, so it gets inherited by the
 		# child process, and this makes sure that it is effect even if the parent
 		# terminates quickly.
-		for s in (signal.SIGHUP,):
-			self.__prev_signals[s] = signal.getsignal(s)
-			signal.signal(s, signal.SIG_IGN)
+		self._rebindSignal(signal.SIGHUP, signal.SIG_IGN)
 
 		try:
 			# Fork a child process so the parent can exit.  This will return control

From dfcf6f1963b87a7fbe87a78f47ba5c0ac40dd558 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 12 Jul 2016 22:53:37 -0400
Subject: [PATCH 128/623] BF+TST: isStarted was not checking __asyncServer for
 exist before checking it

---
 fail2ban/server/server.py        | 4 ++--
 fail2ban/tests/servertestcase.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 7edf19e9..60eea1f3 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -60,7 +60,7 @@ def _thread_name():
 
 class Server:
 	
-	def __init__(self, daemon = False):
+	def __init__(self, daemon=False):
 		self.__loggingLock = Lock()
 		self.__lock = RLock()
 		self.__jails = Jails()
@@ -377,7 +377,7 @@ class Server:
 		return self.__jails[name].actions.getBanTime()
 	
 	def isStarted(self):
-		self.__asyncServer.isActive()
+		return self.__asyncServer is not None and self.__asyncServer.isActive()
 
 	def isAlive(self, jailnum=None):
 		if jailnum is not None and len(self.__jails) != jailnum:
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 9fc62acd..d733da0a 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -158,6 +158,9 @@ class Transmitter(TransmitterBase):
 	def setUp(self):
 		self.server = TestServer()
 		super(Transmitter, self).setUp()
+		# so far isStarted only tested but not used otherwise
+		# and here we don't really .start server
+		self.assertFalse(self.server.isStarted())
 
 	def testStopServer(self):
 		self.assertEqual(self.transm.proceed(["stop"]), (0, None))
@@ -1004,6 +1007,7 @@ class LoggingTests(LogCaptureTestCase):
 		server = TestServer()
 		try:
 			server.start(sock_name, pidfile_name, force=False)
+			self.assertFalse(server.isStarted())
 			self.assertLogged("Server already running")
 		finally:
 			server.quit()

From ad393fcfbbdb0c491e86f891c6ee194fa0134f79 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 13 Jul 2016 09:36:11 +0200
Subject: [PATCH 129/623] outsource assert..isStarted from setup into separate
 test case

---
 fail2ban/tests/servertestcase.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d733da0a..ab606dbe 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -158,6 +158,8 @@ class Transmitter(TransmitterBase):
 	def setUp(self):
 		self.server = TestServer()
 		super(Transmitter, self).setUp()
+
+	def testServerIsNotStarted(self):
 		# so far isStarted only tested but not used otherwise
 		# and here we don't really .start server
 		self.assertFalse(self.server.isStarted())

From 53da35ea49b99a152f68bd4cf46efbcbea22cdb5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 13 Jul 2016 09:44:32 +0200
Subject: [PATCH 130/623] resolves Christmas-tree in logging (replaced with
 duplex space indentation)

---
 fail2ban/client/fail2banclient.py  | 10 +++++-----
 fail2ban/client/fail2bancmdline.py |  4 ++--
 fail2ban/client/fail2banserver.py  |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index e3797279..4e42680b 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -227,11 +227,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# prepare: read config, check configuration is valid, etc.:
 		if phase is not None:
 			phase['start'] = True
-			logSys.debug('-- client phase %s', phase)
+			logSys.debug('  client phase %s', phase)
 		stream = self.__prepareStartServer()
 		if phase is not None:
 			phase['ready'] = phase['start'] = (True if stream else False)
-			logSys.debug('-- client phase %s', phase)
+			logSys.debug('  client phase %s', phase)
 		if not stream:
 			return False
 		# configure server with config stream:
@@ -248,7 +248,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 
 	def __processCommand(self, cmd):
 		if len(cmd) == 1 and cmd[0] == "start":
-			
+
 			ret = self.__startServer(self._conf["background"])
 			if not ret:
 				return False
@@ -322,7 +322,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
 		logSys.debug("__waitOnServer: %r", (alive, maxtime))
-		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping() 
+		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping()
 		with VisualWait(self._conf["verbose"]) as vis:
 			sltime = 0.0125 / 2
 			while self._alive:
@@ -432,7 +432,7 @@ class _VisualWait:
 		self.pos += self.delta
 		if self.delta > 0:
 			s = " #\x1b[1D" if self.pos > 1 else "# \x1b[2D"
-		else: 
+		else:
 			s = "\x1b[1D# \x1b[2D"
 		sys.stdout.write(s)
 		sys.stdout.flush()
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index aadf823a..4cb1927e 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -112,7 +112,7 @@ class Fail2banCmdLine():
 		output("    --timeout               timeout to wait for the server (for internal usage only, don't read configuration)")
 		output("    -h, --help              display this help message")
 		output("    -V, --version           print the version")
-		
+
 		if not caller.endswith('server'):
 			output("")
 			output("Command:")
@@ -185,7 +185,7 @@ class Fail2banCmdLine():
 			if ret is not None:
 				return ret
 
-			logSys.debug("-- conf: %r, args: %r", self._conf, self._args)
+			logSys.debug("  conf: %r, args: %r", self._conf, self._args)
 
 			if initial and PRODUCTION: # pragma: no cover - can't test
 				verbose = self._conf["verbose"]
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index a511e017..9f825bf1 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -46,7 +46,7 @@ class Fail2banServer(Fail2banCmdLine):
 
 	@staticmethod
 	def startServerDirect(conf, daemon=True):
-		logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon)
+		logSys.debug("  direct starting of server in %s, deamon: %s", os.getpid(), daemon)
 		from ..server.server import Server
 		server = None
 		try:
@@ -54,7 +54,7 @@ class Fail2banServer(Fail2banCmdLine):
 			# server object will internally fork self if daemon is True
 			server = Server(daemon)
 			server.start(conf["socket"],
-							conf["pidfile"], conf["force"], 
+							conf["pidfile"], conf["force"],
 							conf=conf)
 		except Exception as e: # pragma: no cover
 			try:
@@ -79,7 +79,7 @@ class Fail2banServer(Fail2banCmdLine):
 		frk = not conf["async"] and PRODUCTION
 		if frk: # pragma: no cover
 			pid = os.fork()
-		logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid)
+		logSys.debug("  async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid)
 		if pid == 0:
 			args = list()
 			args.append(SERVER)

From f0b2cd2e395a4b8e077d71f54d19d6e885da1f37 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 13 Jul 2016 09:54:54 +0200
Subject: [PATCH 131/623] remove no more necessary PRODUCTION from
 fail2banclient.py

---
 fail2ban/client/fail2banclient.py        | 15 ++++++---------
 fail2ban/tests/fail2banclienttestcase.py |  1 -
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 4e42680b..9e6c4bd6 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -35,7 +35,7 @@ from ..version import version
 from .csocket import CSocket
 from .beautifier import Beautifier
 from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \
-	logSys, PRODUCTION, exit, output
+	logSys, exit, output
 
 PROMPT = "fail2ban> "
 
@@ -361,19 +361,16 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 
 			# Interactive mode
 			if self._conf.get("interactive", False):
-				# no readline in test:
-				if PRODUCTION: # pragma: no cover
-					try:
-						import readline
-					except ImportError:
-						raise ServerExecutionException("Readline not available")
+				try:
+					import readline
+				except ImportError:
+					raise ServerExecutionException("Readline not available")
 				try:
 					ret = True
 					if len(args) > 0:
 						ret = self.__processCommand(args)
 					if ret:
-						if PRODUCTION: # pragma: no cover
-							readline.parse_and_bind("tab: complete")
+						readline.parse_and_bind("tab: complete")
 						self.dispInteractive()
 						while True:
 							cmd = input_command()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 2a18d204..ed2d3c46 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -107,7 +107,6 @@ fail2banclient.input_command = _test_input_command
 
 # prevents change logging params, log capturing, etc:
 fail2bancmdline.PRODUCTION = \
-fail2banclient.PRODUCTION = \
 fail2banserver.PRODUCTION = False
 
 

From 41e3f2e5ede8951e4e2a58085718c790aef6477c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 14 Jul 2016 11:35:32 +0200
Subject: [PATCH 132/623] explicit flush stdout/stderr before exit, because of
 possible buffered output in python (echo or grep `fail2ban-client --version`
 can sometimes returns nothing)

---
 fail2ban/client/fail2bancmdline.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 4cb1927e..a14a1cf4 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -270,6 +270,10 @@ class Fail2banCmdLine():
 	@staticmethod
 	def exit(code=0):
 		logSys.debug("Exit with code %s", code)
+		# because of possible buffered output in python, we should flush it before exit:
+		sys.stdout.flush()
+		sys.stderr.flush()
+		# exit
 		Fail2banCmdLine._exit(code)
 
 

From 9b03a6d42e4085b42c9c635d94273a52ee58683b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 14 Jul 2016 12:19:46 +0200
Subject: [PATCH 133/623] 0.10 pre-release: MANIFEST RELEASE and man pages
 updates

---
 MANIFEST                 | 30 ++++++++++------
 RELEASE                  |  2 +-
 man/fail2ban-client.1    | 76 +++++++++++++++++++++++++++-------------
 man/fail2ban-regex.1     | 21 ++++++-----
 man/fail2ban-server.1    | 53 +++++++++++++++++++---------
 man/fail2ban-testcases.1 | 22 +++++++++---
 6 files changed, 138 insertions(+), 66 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index cd250d3d..05e665b2 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -13,9 +13,12 @@ config/action.d/complain.conf
 config/action.d/dshield.conf
 config/action.d/dummy.conf
 config/action.d/firewallcmd-allports.conf
+config/action.d/firewallcmd-common.conf
 config/action.d/firewallcmd-ipset.conf
 config/action.d/firewallcmd-multiport.conf
 config/action.d/firewallcmd-new.conf
+config/action.d/firewallcmd-rich-logging.conf
+config/action.d/firewallcmd-rich-rules.conf
 config/action.d/hostsdeny.conf
 config/action.d/ipfilter.conf
 config/action.d/ipfw.conf
@@ -31,6 +34,7 @@ config/action.d/iptables-new.conf
 config/action.d/iptables-xt_recent-echo.conf
 config/action.d/mail-buffered.conf
 config/action.d/mail.conf
+config/action.d/mail-whois-common.conf
 config/action.d/mail-whois.conf
 config/action.d/mail-whois-lines.conf
 config/action.d/mynetwatchman.conf
@@ -52,6 +56,7 @@ config/action.d/sendmail-whois-ipmatches.conf
 config/action.d/sendmail-whois-lines.conf
 config/action.d/sendmail-whois-matches.conf
 config/action.d/shorewall.conf
+config/action.d/shorewall-ipset-proto6.conf
 config/action.d/smtp.py
 config/action.d/symbiosis-blacklist-allports.conf
 config/action.d/ufw.conf
@@ -67,6 +72,7 @@ config/filter.d/apache-modsecurity.conf
 config/filter.d/apache-nohome.conf
 config/filter.d/apache-noscript.conf
 config/filter.d/apache-overflows.conf
+config/filter.d/apache-pass.conf
 config/filter.d/apache-shellshock.conf
 config/filter.d/assp.conf
 config/filter.d/asterisk.conf
@@ -79,17 +85,18 @@ config/filter.d/cyrus-imap.conf
 config/filter.d/directadmin.conf
 config/filter.d/dovecot.conf
 config/filter.d/dropbear.conf
+config/filter.d/drupal-auth.conf
 config/filter.d/ejabberd-auth.conf
 config/filter.d/exim-common.conf
 config/filter.d/exim.conf
 config/filter.d/exim-spam.conf
 config/filter.d/freeswitch.conf
+config/filter.d/froxlor-auth.conf
 config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
 config/filter.d/haproxy-http-auth.conf
 config/filter.d/horde.conf
-config/filter.d/ignorecommands
 config/filter.d/ignorecommands/apache-fakegooglebot
 config/filter.d/kerio.conf
 config/filter.d/lighttpd-auth.conf
@@ -122,7 +129,6 @@ config/filter.d/selinux-common.conf
 config/filter.d/selinux-ssh.conf
 config/filter.d/sendmail-auth.conf
 config/filter.d/sendmail-reject.conf
-config/filter.d/sendmail-spam.conf
 config/filter.d/sieve.conf
 config/filter.d/sogo-auth.conf
 config/filter.d/solid-pop3d.conf
@@ -148,7 +154,6 @@ config/paths-osx.conf
 CONTRIBUTING.md
 COPYING
 DEVELOP
-doc/run-rootless.txt
 fail2ban-2to3
 fail2ban/client/actionreader.py
 fail2ban/client/beautifier.py
@@ -185,7 +190,6 @@ fail2ban/server/filterpyinotify.py
 fail2ban/server/filtersystemd.py
 fail2ban/server/__init__.py
 fail2ban/server/ipdns.py
-fail2ban/server/iso8601.py
 fail2ban/server/jail.py
 fail2ban/server/jails.py
 fail2ban/server/jailthread.py
@@ -203,21 +207,19 @@ fail2ban/tests/action_d/test_smtp.py
 fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
 fail2ban/tests/banmanagertestcase.py
-fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/clientbeautifiertestcase.py
+fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
 fail2ban/tests/config/filter.d/simple.conf
 fail2ban/tests/config/filter.d/test.conf
 fail2ban/tests/config/filter.d/test.local
+fail2ban/tests/config/filter.d/zzz-generic-example.conf
 fail2ban/tests/config/jail.conf
-fail2ban/tests/config/paths-common.conf
-fail2ban/tests/config/paths-debian.conf
-fail2ban/tests/config/paths-freebsd.conf
-fail2ban/tests/config/paths-osx.conf
 fail2ban/tests/databasetestcase.py
 fail2ban/tests/datedetectortestcase.py
 fail2ban/tests/dummyjail.py
+fail2ban/tests/fail2banclienttestcase.py
 fail2ban/tests/fail2banregextestcase.py
 fail2ban/tests/failmanagertestcase.py
 fail2ban/tests/files/action.d/action_checkainfo.py
@@ -250,13 +252,13 @@ fail2ban/tests/files/ignorecommand.py
 fail2ban/tests/files/logs/3proxy
 fail2ban/tests/files/logs/apache-auth
 fail2ban/tests/files/logs/apache-badbots
-fail2ban/tests/files/logs/apache-botscripts
 fail2ban/tests/files/logs/apache-botsearch
 fail2ban/tests/files/logs/apache-fakegooglebot
 fail2ban/tests/files/logs/apache-modsecurity
 fail2ban/tests/files/logs/apache-nohome
 fail2ban/tests/files/logs/apache-noscript
 fail2ban/tests/files/logs/apache-overflows
+fail2ban/tests/files/logs/apache-pass
 fail2ban/tests/files/logs/apache-shellshock
 fail2ban/tests/files/logs/assp
 fail2ban/tests/files/logs/asterisk
@@ -270,10 +272,12 @@ fail2ban/tests/files/logs/cyrus-imap
 fail2ban/tests/files/logs/directadmin
 fail2ban/tests/files/logs/dovecot
 fail2ban/tests/files/logs/dropbear
+fail2ban/tests/files/logs/drupal-auth
 fail2ban/tests/files/logs/ejabberd-auth
 fail2ban/tests/files/logs/exim
 fail2ban/tests/files/logs/exim-spam
 fail2ban/tests/files/logs/freeswitch
+fail2ban/tests/files/logs/froxlor-auth
 fail2ban/tests/files/logs/groupoffice
 fail2ban/tests/files/logs/gssftpd
 fail2ban/tests/files/logs/guacamole
@@ -309,7 +313,6 @@ fail2ban/tests/files/logs/screensharingd
 fail2ban/tests/files/logs/selinux-ssh
 fail2ban/tests/files/logs/sendmail-auth
 fail2ban/tests/files/logs/sendmail-reject
-fail2ban/tests/files/logs/sendmail-spam
 fail2ban/tests/files/logs/sieve
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
@@ -325,6 +328,7 @@ fail2ban/tests/files/logs/vsftpd
 fail2ban/tests/files/logs/webmin-auth
 fail2ban/tests/files/logs/wuftpd
 fail2ban/tests/files/logs/xinetd-fail
+fail2ban/tests/files/logs/zzz-generic-example
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log
@@ -356,6 +360,8 @@ files/gentoo-confd
 files/gentoo-initd
 files/ipmasq-ZZZzzz_fail2ban.rul
 files/logwatch/fail2ban
+files/logwatch/fail2ban-0.8.log
+files/logwatch/fail2ban-0.9.log
 files/macosx-initd
 files/monit/fail2ban
 files/nagios/check_fail2ban
@@ -373,6 +379,8 @@ man/fail2ban-regex.1
 man/fail2ban-regex.h2m
 man/fail2ban-server.1
 man/fail2ban-server.h2m
+man/fail2ban-testcases.1
+man/fail2ban-testcases.h2m
 man/generate-man
 man/jail.conf.5
 README.md
diff --git a/RELEASE b/RELEASE
index fa89fc34..e570c9eb 100644
--- a/RELEASE
+++ b/RELEASE
@@ -190,7 +190,7 @@ Post Release
 
 Add the following to the top of the ChangeLog::
 
-  ver. 0.9.6 (2016/XX/XXX) - wanna-be-released
+  ver. 0.10.0 (2016/XX/XXX) - wanna-be-released
   -----------
   
   - Fixes:
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index ec5db2d2..0da5fb02 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.2.
-.TH FAIL2BAN-CLIENT "1" "March 2016" "fail2ban-client v0.9.4" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
+.TH FAIL2BAN-CLIENT "1" "July 2016" "fail2ban-client v0.10.0a1" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
-[\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
+[\fIOPTIONS\fR] \fI<COMMAND>\fR
 .SH DESCRIPTION
-Fail2Ban v0.9.4 reads log file that contains password failure report
+Fail2Ban v0.10.0a1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
@@ -19,6 +19,13 @@ socket path
 \fB\-p\fR <FILE>
 pidfile path
 .TP
+\fB\-\-loglevel\fR <LEVEL>
+logging level
+.HP
+\fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
+.HP
+\fB\-\-syslogsocket\fR auto|<FILE>
+.TP
 \fB\-d\fR
 dump configuration. For debugging
 .TP
@@ -38,7 +45,13 @@ force execution of the server (remove socket file)
 start server in background (default)
 .TP
 \fB\-f\fR
-start server in foreground (note that the client forks once itself)
+start server in foreground
+.TP
+\fB\-\-async\fR
+start server in async mode (for internal usage only, don't read configuration)
+.TP
+\fB\-\-timeout\fR
+timeout to wait for the server (for internal usage only, don't read configuration)
 .TP
 \fB\-h\fR, \fB\-\-help\fR
 display this help message
@@ -52,8 +65,12 @@ BASIC
 \fBstart\fR
 starts the server and the jails
 .TP
+\fBrestart\fR
+restarts the server
+.TP
 \fBreload\fR
-reloads the configuration
+reloads the configuration without
+restart
 .TP
 \fBreload <JAIL>\fR
 reloads the jail <JAIL>
@@ -69,6 +86,10 @@ server
 \fBping\fR
 tests if the server is alive
 .TP
+\fBecho\fR
+for internal usage, returns back
+and outputs a given string
+.TP
 \fBhelp\fR
 return this output
 .TP
@@ -227,8 +248,9 @@ for <JAIL>
 \fBset <JAIL> maxlines <LINES>\fR
 sets the number of <LINES> to
 buffer for regex search for <JAIL>
-.TP
-\fBset <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]\fR
+.IP
+set <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]
+.IP
 adds a new action named <ACT> for
 <JAIL>. Optionally for a Python
 based action, a <PYTHONFILE> and
@@ -240,38 +262,45 @@ removes the action <ACT> from
 <JAIL>
 .IP
 COMMAND ACTION CONFIGURATION
-.TP
-\fBset <JAIL> action <ACT> actionstart <CMD>\fR
+.IP
+set <JAIL> action <ACT> actionstart <CMD>
+.IP
 sets the start command <CMD> of
 the action <ACT> for <JAIL>
-.TP
-\fBset <JAIL> action <ACT> actionstop <CMD> sets the stop command <CMD> of the\fR
+.IP
+set <JAIL> action <ACT> actionstop <CMD> sets the stop command <CMD> of the
+.IP
 action <ACT> for <JAIL>
-.TP
-\fBset <JAIL> action <ACT> actioncheck <CMD>\fR
+.IP
+set <JAIL> action <ACT> actioncheck <CMD>
+.IP
 sets the check command <CMD> of
 the action <ACT> for <JAIL>
 .TP
 \fBset <JAIL> action <ACT> actionban <CMD>\fR
 sets the ban command <CMD> of the
 action <ACT> for <JAIL>
-.TP
-\fBset <JAIL> action <ACT> actionunban <CMD>\fR
+.IP
+set <JAIL> action <ACT> actionunban <CMD>
+.IP
 sets the unban command <CMD> of
 the action <ACT> for <JAIL>
-.TP
-\fBset <JAIL> action <ACT> timeout <TIMEOUT>\fR
+.IP
+set <JAIL> action <ACT> timeout <TIMEOUT>
+.IP
 sets <TIMEOUT> as the command
 timeout in seconds for the action
 <ACT> for <JAIL>
 .IP
 GENERAL ACTION CONFIGURATION
-.TP
-\fBset <JAIL> action <ACT> <PROPERTY> <VALUE>\fR
+.IP
+set <JAIL> action <ACT> <PROPERTY> <VALUE>
+.IP
 sets the <VALUE> of <PROPERTY> for
 the action <ACT> for <JAIL>
-.TP
-\fBset <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]\fR
+.IP
+set <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]
+.IP
 calls the <METHOD> with
 <JSONKWARGS> for the action <ACT>
 for <JAIL>
@@ -376,9 +405,6 @@ gets the value of <PROPERTY> for
 the action <ACT> for <JAIL>
 .SH FILES
 \fI/etc/fail2ban/*\fR
-.SH AUTHOR
-Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>.
-Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>.
 .SH "REPORTING BUGS"
 Report bugs to https://github.com/fail2ban/fail2ban/issues
 .SH COPYRIGHT
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 740b461c..f954cb3a 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.2.
-.TH FAIL2BAN-REGEX "1" "March 2016" "fail2ban-regex 0.9.4" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
+.TH FAIL2BAN-REGEX "1" "July 2016" "fail2ban-regex 0.10.0a1" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
 .B fail2ban-regex
-[\fI\,OPTIONS\/\fR] \fI\,<LOG> <REGEX> \/\fR[\fI\,IGNOREREGEX\/\fR]
+[\fIOPTIONS\fR] \fI<LOG> <REGEX> \fR[\fIIGNOREREGEX\fR]
 .SH DESCRIPTION
 Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
@@ -16,7 +16,7 @@ string
 a string representing a log line
 .TP
 filename
-path to a log file (\fI\,/var/log/auth.log\/\fP)
+path to a log file (\fI/var/log/auth.log\fP)
 .TP
 "systemd\-journal"
 search systemd journal (systemd\-python required)
@@ -42,20 +42,23 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR
+\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fIDATEPATTERN\fR
 set custom pattern used to match date/times
 .TP
-\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fI\,ENCODING\/\fR
+\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fIENCODING\fR
 File encoding. Default: system locale
 .TP
-\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fI\,MAXLINES\/\fR
+\fB\-r\fR, \fB\-\-raw\fR
+Raw hosts, don't resolve dns
+.TP
+\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fIMAXLINES\fR
 maxlines for multi\-line regex
 .TP
-\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fI\,JOURNALMATCH\/\fR
+\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fIJOURNALMATCH\fR
 journalctl style matches overriding filter file.
 "systemd\-journal" only
 .TP
-\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR
+\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fILOG_LEVEL\fR
 Log level for the Fail2Ban logger to use
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 90ec1d80..3b970024 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,24 +1,17 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.2.
-.TH FAIL2BAN-SERVER "1" "March 2016" "fail2ban-server v0.9.4" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
+.TH FAIL2BAN-SERVER "1" "July 2016" "fail2ban-server v0.10.0a1" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
-[\fI\,OPTIONS\/\fR]
+[\fIOPTIONS\fR]
 .SH DESCRIPTION
-Fail2Ban v0.9.4 reads log file that contains password failure report
+Fail2Ban v0.10.0a1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
-.PP
-Only use this command for debugging purpose. Start the server with
-fail2ban\-client instead. The default behaviour is to start the server
-in background.
 .SH OPTIONS
 .TP
-\fB\-b\fR
-start in background
-.TP
-\fB\-f\fR
-start in foreground
+\fB\-c\fR <DIR>
+configuration directory
 .TP
 \fB\-s\fR <FILE>
 socket path
@@ -26,17 +19,45 @@ socket path
 \fB\-p\fR <FILE>
 pidfile path
 .TP
+\fB\-\-loglevel\fR <LEVEL>
+logging level
+.HP
+\fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
+.HP
+\fB\-\-syslogsocket\fR auto|<FILE>
+.TP
+\fB\-d\fR
+dump configuration. For debugging
+.TP
+\fB\-i\fR
+interactive mode
+.TP
+\fB\-v\fR
+increase verbosity
+.TP
+\fB\-q\fR
+decrease verbosity
+.TP
 \fB\-x\fR
 force execution of the server (remove socket file)
 .TP
+\fB\-b\fR
+start server in background (default)
+.TP
+\fB\-f\fR
+start server in foreground
+.TP
+\fB\-\-async\fR
+start server in async mode (for internal usage only, don't read configuration)
+.TP
+\fB\-\-timeout\fR
+timeout to wait for the server (for internal usage only, don't read configuration)
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 display this help message
 .TP
 \fB\-V\fR, \fB\-\-version\fR
 print the version
-.SH AUTHOR
-Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>.
-Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>.
 .SH "REPORTING BUGS"
 Report bugs to https://github.com/fail2ban/fail2ban/issues
 .SH COPYRIGHT
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index dc0fee32..9089d1ed 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.2.
-.TH FAIL2BAN-TESTCASES "1" "March 2016" "fail2ban-testcases 0.9.4" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
+.TH FAIL2BAN-TESTCASES "1" "July 2016" "fail2ban-testcases 0.10.0a1" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS
 .B fail2ban-testcases
-[\fI\,OPTIONS\/\fR] [\fI\,regexps\/\fR]
+[\fIOPTIONS\fR] [\fIregexps\fR]
 .SH DESCRIPTION
 Script to run Fail2Ban tests battery
 .SH OPTIONS
@@ -15,12 +15,26 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR
+\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fILOG_LEVEL\fR
 Log level for the logger to use during running tests
 .TP
 \fB\-n\fR, \fB\-\-no\-network\fR
 Do not run tests that require the network
 .TP
+\fB\-g\fR, \fB\-\-no\-gamin\fR
+Do not run tests that require the gamin
+.TP
+\fB\-m\fR, \fB\-\-memory\-db\fR
+Run database tests using memory instead of file
+.TP
+\fB\-f\fR, \fB\-\-fast\fR
+Try to increase speed of the tests, decreasing of wait
+intervals, memory database
+.TP
+\fB\-i\fR, \fB\-\-ignore\fR
+negate [regexps] filter to ignore tests matched
+specified regexps
+.TP
 \fB\-t\fR, \fB\-\-log\-traceback\fR
 Enrich log\-messages with compressed tracebacks
 .TP

From 5797ea0ae2932ed5f752508da1f49bd76356e537 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 14 Jul 2016 09:37:39 -0400
Subject: [PATCH 134/623] Let's work toward a2 release

---
 fail2ban/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/version.py b/fail2ban/version.py
index db4daa97..48d2ed66 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.0a1"
+version = "0.10.0a2"

From c15e968c1fda97219e27ef210fc146d7668b1f27 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 14 Jul 2016 21:17:18 -0400
Subject: [PATCH 135/623] DOC: added a note about choose correct branch for PR

---
 .github/PULL_REQUEST_TEMPLATE.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index a9fff350..3a17ccc2 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,9 +1,11 @@
 Before submitting your PR, please review the following checklist:
 
+- [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement
+      against 0.9.x series, choose `master` branch
 - [ ] **CONSIDER adding a unit test** if your PR resolves an issue
 - [ ] **LIST ISSUES** this PR resolves
-- [ ] **MAKE SURE** this PR doesn't break existing tests 
+- [ ] **MAKE SURE** this PR doesn't break existing tests
 - [ ] **KEEP PR small** so it could be easily reviewed.
 - [ ] **AVOID** making unnecessary stylistic changes in unrelated code
-- [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines 
-      within `fail2ban/tests/files/logs/X` file
\ No newline at end of file
+- [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines
+      within `fail2ban/tests/files/logs/X` file

From 9d56079756ca6aef901a69041786ab137d0b5ae3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 25 Jul 2016 19:12:33 +0200
Subject: [PATCH 136/623] several test cases rewritten using new assertIn,
 assertNotIn (better as own from unittest, because support generators
 beautifying, etc.) + new forward compatibility method assertRaisesRegexp; +
 methods assertIn, assertNotIn, assertRaisesRegexp are test covered now; +
 easy-fix for distributions compatible test cases (e.g. fedora default backend
 is 'systemd'), (closes gh-1353, closes gh-1490)

cherry pick into 0.9 branch
---
 fail2ban/tests/action_d/test_badips.py |  2 +-
 fail2ban/tests/action_d/test_smtp.py   |  8 +--
 fail2ban/tests/actionstestcase.py      |  6 +-
 fail2ban/tests/clientreadertestcase.py | 18 +++---
 fail2ban/tests/databasetestcase.py     | 10 +--
 fail2ban/tests/filtertestcase.py       |  4 +-
 fail2ban/tests/misctestcase.py         | 87 ++++++++++++++++++++++++--
 fail2ban/tests/servertestcase.py       |  6 +-
 fail2ban/tests/utils.py                | 50 ++++++++++++---
 9 files changed, 151 insertions(+), 40 deletions(-)

diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 97dabada..2b9090cc 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -51,7 +51,7 @@ if sys.version_info >= (2,7):
 
 		def testCategory(self):
 			categories = self.action.getCategories()
-			self.assertTrue("ssh" in categories)
+			self.assertIn("ssh", categories)
 			self.assertTrue(len(categories) >= 10)
 
 			self.assertRaises(
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 1385fe82..5c8b1923 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -104,21 +104,21 @@ class SMTPActionTest(unittest.TestCase):
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		subject = "Subject: [Fail2Ban] %s: banned %s" % (
 			self.jail.name, aInfo['ip'])
-		self.assertTrue(subject in self.smtpd.data.replace("\n", ""))
+		self.assertIn(subject, self.smtpd.data.replace("\n", ""))
 		self.assertTrue(
 			"%i attempts" % aInfo['failures'] in self.smtpd.data)
 
 		self.action.matches = "matches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['matches'] in self.smtpd.data)
+		self.assertIn(aInfo['matches'], self.smtpd.data)
 
 		self.action.matches = "ipjailmatches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['ipjailmatches'] in self.smtpd.data)
+		self.assertIn(aInfo['ipjailmatches'], self.smtpd.data)
 
 		self.action.matches = "ipmatches"
 		self.action.ban(aInfo)
-		self.assertTrue(aInfo['ipmatches'] in self.smtpd.data)
+		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
 
 	def testOptions(self):
 		self.action.start()
diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py
index dff0d4c6..8969db36 100644
--- a/fail2ban/tests/actionstestcase.py
+++ b/fail2ban/tests/actionstestcase.py
@@ -66,12 +66,12 @@ class ExecuteActions(LogCaptureTestCase):
 	def testActionsManipulation(self):
 		self.__actions.add('test')
 		self.assertTrue(self.__actions['test'])
-		self.assertTrue('test' in self.__actions)
-		self.assertFalse('nonexistant action' in self.__actions)
+		self.assertIn('test', self.__actions)
+		self.assertNotIn('nonexistant action', self.__actions)
 		self.__actions.add('test1')
 		del self.__actions['test']
 		del self.__actions['test1']
-		self.assertFalse('test' in self.__actions)
+		self.assertNotIn('test', self.__actions)
 		self.assertEqual(len(self.__actions), 0)
 
 		self.__actions.setBanTime(127)
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 521fda2e..39fd0daa 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -542,12 +542,12 @@ class JailsReaderTest(LogCaptureTestCase):
 				self.assertTrue(actionReader.read())
 				actionReader.getOptions({})	  # populate _opts
 				if not actionName.endswith('-common'):
-					self.assertTrue('Definition' in actionReader.sections(),
+					self.assertIn('Definition', actionReader.sections(),
 						msg="Action file %r is lacking [Definition] section" % actionConfig)
 					# all must have some actionban defined
 					self.assertTrue(actionReader._opts.get('actionban', '').strip(),
 						msg="Action file %r is lacking actionban" % actionConfig)
-				self.assertTrue('Init' in actionReader.sections(),
+				self.assertIn('Init', actionReader.sections(),
 						msg="Action file %r is lacking [Init] section" % actionConfig)
 
 		def testReadStockJailConf(self):
@@ -599,7 +599,7 @@ class JailsReaderTest(LogCaptureTestCase):
 					self.assertTrue(len(actName))
 					self.assertTrue(isinstance(actOpt, dict))
 					if actName == 'iptables-multiport':
-						self.assertTrue('port' in actOpt)
+						self.assertIn('port', actOpt)
 
 					actionReader = ActionReader(actName, jail, {}, 
 						share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
@@ -649,11 +649,13 @@ class JailsReaderTest(LogCaptureTestCase):
 
 			# and we know even some of them by heart
 			for j in ['sshd', 'recidive']:
-				# by default we have 'auto' backend ATM
-				self.assertTrue(['add', j, 'auto'] in comm_commands)
+				# by default we have 'auto' backend ATM, but some distributions can overwrite it, 
+				# (e.g. fedora default is 'systemd') therefore let check it without backend...
+				self.assertIn(['add', j], 
+					(cmd[:2] for cmd in comm_commands if len(cmd) == 3 and cmd[0] == 'add'))
 				# and warn on useDNS
-				self.assertTrue(['set', j, 'usedns', 'warn'] in comm_commands)
-				self.assertTrue(['start', j] in comm_commands)
+				self.assertIn(['set', j, 'usedns', 'warn'], comm_commands)
+				self.assertIn(['start', j], comm_commands)
 
 			# last commands should be the 'start' commands
 			self.assertEqual(comm_commands[-1][0], 'start')
@@ -672,7 +674,7 @@ class JailsReaderTest(LogCaptureTestCase):
 					action_name = action.getName()
 					if '<blocktype>' in str(commands):
 						# Verify that it is among cInfo
-						self.assertTrue('blocktype' in action._initOpts)
+						self.assertIn('blocktype', action._initOpts)
 						# Verify that we have a call to set it up
 						blocktype_present = False
 						target_command = [jail_name, 'action', action_name]
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 5d83710f..d039c920 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -145,7 +145,7 @@ class DatabaseTest(LogCaptureTestCase):
 
 		self.db.addLog(self.jail, self.fileContainer)
 
-		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+		self.assertIn(filename, self.db.getLogPaths(self.jail))
 		os.remove(filename)
 
 	def testUpdateLog(self):
@@ -376,17 +376,17 @@ class DatabaseTest(LogCaptureTestCase):
 		# Delete jail (just disabled it):
 		self.db.delJail(self.jail)
 		jails = self.db.getJailNames()
-		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
 		jails = self.db.getJailNames(enabled=False)
-		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
 		jails = self.db.getJailNames(enabled=True)
 		self.assertTrue(len(jails) == 0)
 		# Add it again - should just enable it:
 		self.db.addJail(self.jail)
 		jails = self.db.getJailNames()
-		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
 		jails = self.db.getJailNames(enabled=True)
-		self.assertTrue(len(jails) == 1 and self.jail.name in jails)
+		self.assertIn(len(jails) == 1 and self.jail.name, jails)
 		jails = self.db.getJailNames(enabled=False)
 		self.assertTrue(len(jails) == 0)
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index dd3468ed..7907965b 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1320,11 +1320,11 @@ class DNSUtilsTests(unittest.TestCase):
 		for i in xrange(5):
 			c.set(i, i)
 		self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)])
-		self.assertFalse(-1 in [c.get(i, -1) for i in xrange(5)])
+		self.assertNotIn(-1, (c.get(i, -1) for i in xrange(5)))
 		# add one - too many:
 		c.set(10, i)
 		# one element should be removed :
-		self.assertTrue(-1 in [c.get(i, -1) for i in xrange(5)])
+		self.assertIn(-1, (c.get(i, -1) for i in xrange(5)))
 		# test max size (not expired):
 		for i in xrange(10):
 			c.set(i, 1)
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index f72d9316..df347e77 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -23,6 +23,7 @@ __license__ = "GPL"
 
 import logging
 import os
+import re
 import sys
 import unittest
 import tempfile
@@ -32,6 +33,8 @@ import datetime
 from glob import glob
 from StringIO import StringIO
 
+from utils import LogCaptureTestCase, logSys as DefLogSys
+
 from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger
 from ..helpers import splitwords
 from ..server.datetemplate import DatePatternRegex
@@ -149,7 +152,7 @@ class SetupTest(unittest.TestCase):
 					  % (sys.executable, self.setup))
 
 
-class TestsUtilsTest(unittest.TestCase):
+class TestsUtilsTest(LogCaptureTestCase):
 
 	def testmbasename(self):
 		self.assertEqual(mbasename("sample.py"), 'sample')
@@ -184,12 +187,88 @@ class TestsUtilsTest(unittest.TestCase):
 			if not ('fail2ban-testcases' in s):
 				# we must be calling it from setup or nosetests but using at least
 				# nose's core etc
-				self.assertTrue('>' in s, msg="no '>' in %r" % s)
+				self.assertIn('>', s)
 			elif not ('coverage' in s):
 				# There is only "fail2ban-testcases" in this case, no true traceback
-				self.assertFalse('>' in s, msg="'>' present in %r" % s)
+				self.assertNotIn('>', s)
 
-			self.assertTrue(':' in s, msg="no ':' in %r" % s)
+			self.assertIn(':', s)
+
+	def _testAssertionErrorRE(self, regexp, fun, *args, **kwargs):
+		self.assertRaisesRegexp(AssertionError, regexp, fun, *args, **kwargs)
+	
+	def testExtendedAssertRaisesRE(self):
+		## test _testAssertionErrorRE several fail cases:
+		def _key_err(msg):
+			raise KeyError(msg)			
+		self.assertRaises(KeyError,
+			self._testAssertionErrorRE, r"^failed$", 
+				_key_err, 'failed')
+		self.assertRaises(AssertionError,
+			self._testAssertionErrorRE, r"^failed$",
+				self.fail, '__failed__')
+		self._testAssertionErrorRE(r'failed.* does not match .*__failed__',
+			lambda: self._testAssertionErrorRE(r"^failed$",
+				self.fail, '__failed__')
+		)
+		## no exception in callable:
+		self.assertRaises(AssertionError,
+			self._testAssertionErrorRE, r"", int, 1)
+		self._testAssertionErrorRE(r'0 AssertionError not raised X.* does not match .*AssertionError not raised',
+			lambda: self._testAssertionErrorRE(r"^0 AssertionError not raised X$",
+				lambda: self._testAssertionErrorRE(r"", int, 1))
+		)
+
+	def testExtendedAssertMethods(self):
+		## assertIn, assertNotIn positive case:
+		self.assertIn('a', ['a', 'b', 'c', 'd'])
+		self.assertIn('a', ('a', 'b', 'c', 'd',))
+		self.assertIn('a', 'cba')
+		self.assertIn('a', (c for c in 'cba' if c != 'b'))
+		self.assertNotIn('a', ['b', 'c', 'd'])
+		self.assertNotIn('a', ('b', 'c', 'd',))
+		self.assertNotIn('a', 'cbd')
+		self.assertNotIn('a', (c.upper() for c in 'cba' if c != 'b'))
+		## assertIn, assertNotIn negative case:
+		self._testAssertionErrorRE(r"'a' unexpectedly found in 'cba'",
+			self.assertNotIn, 'a', 'cba')
+		self._testAssertionErrorRE(r"1 unexpectedly found in \[0, 1, 2\]",
+			self.assertNotIn, 1, xrange(3))
+		self._testAssertionErrorRE(r"'A' unexpectedly found in \['C', 'A'\]",
+			self.assertNotIn, 'A', (c.upper() for c in 'cba' if c != 'b'))
+		self._testAssertionErrorRE(r"'a' was not found in 'xyz'",
+			self.assertIn, 'a', 'xyz')
+		self._testAssertionErrorRE(r"5 was not found in \[0, 1, 2\]",
+			self.assertIn, 5, xrange(3))
+		self._testAssertionErrorRE(r"'A' was not found in \['C', 'B'\]",
+			self.assertIn, 'A', (c.upper() for c in 'cba' if c != 'a'))
+		## assertLogged, assertNotLogged positive case:
+		logSys = DefLogSys
+		self.pruneLog()
+		logSys.debug('test "xyz"')
+		self.assertLogged('test "xyz"')
+		self.assertLogged('test', 'xyz', all=True)
+		self.assertNotLogged('test', 'zyx', all=False)
+		self.assertNotLogged('test_zyx', 'zyx', all=True)
+		self.assertLogged('test', 'zyx', all=False)
+		self.pruneLog()
+		logSys.debug('xxxx "xxx"')
+		self.assertNotLogged('test "xyz"')
+		self.assertNotLogged('test', 'xyz', all=False)
+		self.assertNotLogged('test', 'xyz', 'zyx', all=True)
+		## assertLogged, assertNotLogged negative case:
+		self.pruneLog()
+		logSys.debug('test "xyz"')
+		self._testAssertionErrorRE(r"All of the .* were found present in the log",
+			self.assertNotLogged, 'test "xyz"')
+		self._testAssertionErrorRE(r"was found in the log",
+			self.assertNotLogged, 'test', 'xyz', all=True)
+		self._testAssertionErrorRE(r"was not found in the log",
+			self.assertLogged, 'test', 'zyx', all=True)
+		self._testAssertionErrorRE(r"None among .* was found in the log",
+			self.assertLogged, 'test_zyx', 'zyx', all=False)
+		self._testAssertionErrorRE(r"All of the .* were found present in the log",
+			self.assertNotLogged, 'test', 'xyz', all=False)
 
 	def testFormatterWithTraceBack(self):
 		strout = StringIO()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ab606dbe..6c0f46cf 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -252,7 +252,7 @@ class Transmitter(TransmitterBase):
 			3) )
 		self.assertEqual(
 			self.transm.proceed(["stop", self.jailName]), (0, None))
-		self.assertTrue(self.jailName not in self.server._Server__jails)
+		self.assertNotIn(self.jailName, self.server._Server__jails)
 
 	def testStartStopAllJail(self):
 		self.server.addJail("TestJail2", FAST_BACKEND)
@@ -269,8 +269,8 @@ class Transmitter(TransmitterBase):
 			3) )
 		self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None))
 		self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) )
-		self.assertTrue(self.jailName not in self.server._Server__jails)
-		self.assertTrue("TestJail2" not in self.server._Server__jails)
+		self.assertNotIn(self.jailName, self.server._Server__jails)
+		self.assertNotIn("TestJail2", self.server._Server__jails)
 
 	def testJailIdle(self):
 		self.assertEqual(
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index a06fbef0..2838fa05 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -22,6 +22,7 @@ __author__ = "Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko"
 __license__ = "GPL"
 
+import itertools
 import logging
 import optparse
 import os
@@ -323,16 +324,45 @@ def gatherTests(regexps=None, opts=None):
 	return tests
 
 
-# forwards compatibility of unittest.TestCase for some early python versions
-if not hasattr(unittest.TestCase, 'assertIn'):
-	def __assertIn(self, a, b, msg=None):
-		if a not in b: # pragma: no cover
-			self.fail(msg or "%r was not found in %r" % (a, b))
-	unittest.TestCase.assertIn = __assertIn
-	def __assertNotIn(self, a, b, msg=None):
-		if a in b: # pragma: no cover
-			self.fail(msg or "%r was found in %r" % (a, b))
-	unittest.TestCase.assertNotIn = __assertNotIn
+#
+# Forwards compatibility of unittest.TestCase for some early python versions
+#
+
+if not hasattr(unittest.TestCase, 'assertRaisesRegexp'):
+	def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs):
+		try:
+			fun(*args, **kwargs)
+		except exccls as e:
+			if re.search(regexp, e.message) is None:
+				self.fail('\"%s\" does not match \"%s\"' % (regexp, e.message))
+		else:
+			self.fail('%s not raised' % getattr(exccls, '__name__'))
+	unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp
+
+# always custom following methods, because we use atm better version of both (support generators)
+if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
+	def assertIn(self, a, b, msg=None):
+		bb = b
+		wrap = False
+		if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring):
+			b, bb = itertools.tee(b)
+			wrap = True
+		if a not in b:
+			if wrap: bb = list(bb)
+			msg = msg or "%r was not found in %r" % (a, bb)
+			self.fail(msg)
+	unittest.TestCase.assertIn = assertIn
+	def assertNotIn(self, a, b, msg=None):
+		bb = b
+		wrap = False
+		if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring):
+			b, bb = itertools.tee(b)
+			wrap = True
+		if a in b:
+			if wrap: bb = list(bb)
+			msg = msg or "%r unexpectedly found in %r" % (a, bb)
+			self.fail(msg)
+	unittest.TestCase.assertNotIn = assertNotIn
 
 
 class LogCaptureTestCase(unittest.TestCase):

From f5b7ffeb4b111952e639694f6786a47439a3b160 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 1 Aug 2016 15:18:59 +0200
Subject: [PATCH 137/623] DOC: Reformatted ChangeLog into legit Markdown (after
 merge of master to 0.10)

---
 ChangeLog | 99 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4772d908..6583dd14 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -11,57 +11,58 @@ ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
 
 TODO: implementing of options resp. other tasks from PR #1346
 
-- Fixes:
-   * [grave] memory leak's fixed (gh-1277, gh-1234)
-   * tricky bug fix: last position of log file will be never retrieved (gh-795),
-     because of CASCADE all log entries will be deleted from logs table together with jail, 
-     if used "INSERT OR REPLACE" statement
-   * asyncserver (asyncore) code fixed and test cases repaired (again gh-161)
-   * testSocket: sporadical bug repaired - wait for server thread starts a socket (listener)
-   * testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash,
-     kill tree in any case (gh-1155)
+### Fixes
+* [grave] memory leak's fixed (gh-1277, gh-1234)
+* tricky bug fix: last position of log file will be never retrieved (gh-795),
+  because of CASCADE all log entries will be deleted from logs table together with jail, 
+  if used "INSERT OR REPLACE" statement
+* asyncserver (asyncore) code fixed and test cases repaired (again gh-161)
+* testSocket: sporadical bug repaired - wait for server thread starts a socket (listener)
+* testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash,
+  kill tree in any case (gh-1155)
 
-- New Features:
-   * IPv6 support:
-     - IP addresses are now handled as objects rather than strings capable for 
-       handling both address types IPv4 and IPv6
-     - iptables related actions have been amended to support IPv6 specific actions
-       additionally
-     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
-     - pf action for *BSD systems has been improved and supports now also v4 and v6
-     - name resolution is now working for either address type
-		 - new conditional section functionality used in config resp. includes:
-       - [Init?family=inet4] - IPv4 qualified hosts only
-       - [Init?family=inet6] - IPv6 qualified hosts only
+### New Features
+* IPv6 support:
+    - IP addresses are now handled as objects rather than strings capable for 
+      handling both address types IPv4 and IPv6
+    - iptables related actions have been amended to support IPv6 specific actions
+      additionally
+    - hostsdeny and route actions have been tested to be aware of v4 and v6 already
+    - pf action for *BSD systems has been improved and supports now also v4 and v6
+    - name resolution is now working for either address type
+    - new conditional section functionality used in config resp. includes:
+      - [Init?family=inet4] - IPv4 qualified hosts only
+      - [Init?family=inet6] - IPv6 qualified hosts only
+
+### Enhancements
+* huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)
+* datedetector: in-place reordering using hits and last used time: 
+  matchTime, template list etc. rewritten because of performance degradation
+* prevent out of memory situation if many IP's makes extremely many failures (maxEntries)
+* introduced string to seconds (str2seconds) for configuration entries with time,
+  use `1h` instead of `3600`, `1d` instead of `86400`, etc
+* seekToTime - prevent completely read of big files first time (after start of service), 
+  initial seek to start time using half-interval search algorithm (see issue gh-795)
+* ticket and some other modules prepared to easy merge with newest version of 'ban-time-incr'
+* cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name,
+  especially for wrong dns or lazy dns-system
+* FailManager memory-optimization: increases performance, 
+  prevents memory leakage, because don't copy failures list on some operations
+* fail2ban-testcases - new options introduced:
+    - `-f`, `--fast` to decrease wait intervals, avoid passive waiting, and skip
+      few very slow test cases (implied memory database, see `-m` and no gamin tests `-g`)
+    - `-g`, `--no-gamin` to prevent running of tests that require the gamin (slow)
+    - `-m`, `--memory-db` - run database tests using memory instead of file
+    - `-i`, `--ignore` - negate [regexps] filter to ignore tests matched specified regexps
+* background servicing: prevents memory leak on some platforms/python versions, using forced GC
+  in periodic intervals (latency and threshold)
+* executeCmd partially moved from action to new module utils
+* several functionality of class `DNSUtils` moved to new class `IPAddr`, 
+  both classes moved to new module `ipdns`
+* pseudo-conditional section introduced, for conditional substitution resp. 
+  evaluation of parameters for different family qualified hosts, 
+  syntax `[Section?family=inet6]` (currently use for IPv6-support only).
 
-- Enhancements:
-   * huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)
-   * datedetector: in-place reordering using hits and last used time: 
-     matchTime, template list etc. rewritten because of performance degradation
-   * prevent out of memory situation if many IP's makes extremely many failures (maxEntries)
-   * introduced string to seconds (str2seconds) for configuration entries with time,
-     use `1h` instead of `3600`, `1d` instead of `86400`, etc
-   * seekToTime - prevent completely read of big files first time (after start of service), 
-     initial seek to start time using half-interval search algorithm (see issue gh-795)
-   * ticket and some other modules prepared to easy merge with newest version of 'ban-time-incr'
-   * cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name,
-     especially for wrong dns or lazy dns-system
-   * FailManager memory-optimization: increases performance, 
-     prevents memory leakage, because don't copy failures list on some operations
-   * fail2ban-testcases - new options introduced:
-     - `-f`, `--fast` to decrease wait intervals, avoid passive waiting, and skip
-       few very slow test cases (implied memory database, see `-m` and no gamin tests `-g`)
-     - `-g`, `--no-gamin` to prevent running of tests that require the gamin (slow)
-     - `-m`, `--memory-db` - run database tests using memory instead of file
-     - `-i`, `--ignore` - negate [regexps] filter to ignore tests matched specified regexps
-   * background servicing: prevents memory leak on some platforms/python versions, using forced GC
-     in periodic intervals (latency and threshold)
-   * executeCmd partially moved from action to new module utils
-   * several functionality of class `DNSUtils` moved to new class `IPAddr`, 
-     both classes moved to new module `ipdns`
-   * pseudo-conditional section introduced, for conditional substitution resp. 
-     evaluation of parameters for different family qualified hosts, 
-     syntax `[Section?family=inet6]` (currently use for IPv6-support only).
 
 ver. 0.9.5 (2016/07/15) - old-not-obsolete
 -----------

From 04427adb95207f0032d2b919f9690f0d65ef2111 Mon Sep 17 00:00:00 2001
From: maksyms <maksym@hotmail.com>
Date: Mon, 8 Aug 2016 22:07:55 +0100
Subject: [PATCH 138/623] Accept no space after "failed:" (#1501)

yoh: Squashed to ease cherry-picking into 0.9

* accept no space after "failed:"

fix issue #1497

* accept no space after "failed:"

* Update postfix-sasl

* Update postfix-sasl

* Update postfix-sasl
---
 config/filter.d/postfix-sasl.conf      | 2 +-
 fail2ban/tests/files/logs/postfix-sasl | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf
index 4a6ceaaa..1a24ca94 100644
--- a/config/filter.d/postfix-sasl.conf
+++ b/config/filter.d/postfix-sasl.conf
@@ -9,7 +9,7 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
 
-failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/:]*={0,2})?\s*$
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
 
 ignoreregex = authentication failed: Connection lost to authentication server$
 
diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl
index 9fcb0f49..cdcb5121 100644
--- a/fail2ban/tests/files/logs/postfix-sasl
+++ b/fail2ban/tests/files/logs/postfix-sasl
@@ -26,3 +26,7 @@ Jan 29 08:11:45 mail postfix-incoming/smtpd[10752]: warning: unknown[1.1.1.1]: S
 
 # failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" }
 Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: warning: astra4139.startdedicated.de[62.138.2.143]: SASL LOGIN authentication failed: UGFzc3dvcmQ6
+
+# failJSON: { "time": "2005-08-03T15:30:49", "match": true , "host": "98.191.84.74" }
+Aug 3 15:30:49 ksusha postfix/smtpd[17041]: warning: mail.foldsandwalker.com[98.191.84.74]: SASL Plain authentication failed:
+

From ad19c8401b7d3de247caafb4457de8c53f4651c0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Aug 2016 15:00:54 +0200
Subject: [PATCH 139/623] minor fix: modules are not intended to be executed
 directly (amend to #1426)

---
 fail2ban/client/fail2banclient.py  | 1 -
 fail2ban/client/fail2bancmdline.py | 1 -
 fail2ban/client/fail2banserver.py  | 1 -
 3 files changed, 3 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 9e6c4bd6..8be9e085 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index a14a1cf4..311c7af9 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 9f825bf1..147e131f 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -1,4 +1,3 @@
-#!/usr/bin/python
 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
 # vi: set ft=python sts=4 ts=4 sw=4 noet :
 #

From 77f451c4a38f60fabfa58e8b0a17d399bf7c7f05 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Aug 2016 18:34:18 +0200
Subject: [PATCH 140/623] introduces new command "fail2ban-python", as
 automatically created symlink to python executable, where fail2ban currently
 installed (resp. its modules are located); fixed pythonic filters and test
 scripts (running via "fail2ban-python" now); fixed test case
 "testSetupInstallRoot" not for default python (also using direct call, out of
 virtualenv);

---
 bin/fail2ban-testcases                        |  5 ++-
 .../ignorecommands/apache-fakegooglebot       |  2 +-
 fail2ban/helpers.py                           | 18 +++++++++
 .../tests/files/config/apache-auth/digest.py  |  2 +-
 fail2ban/tests/files/ignorecommand.py         |  2 +-
 fail2ban/tests/misctestcase.py                |  6 ++-
 setup.py                                      | 40 ++++++++++++++++++-
 7 files changed, 69 insertions(+), 6 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 98b9118f..dcc55421 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -38,11 +38,14 @@ if os.path.exists("fail2ban/__init__.py"):
 from fail2ban.version import version
 
 from fail2ban.tests.utils import gatherTests
-from fail2ban.helpers import FormatterWithTraceBack, getLogger
+from fail2ban.helpers import updatePyExec, FormatterWithTraceBack, getLogger
 from fail2ban.server.mytime import MyTime
 
 from optparse import OptionParser, Option
 
+# Update fail2ban-python env to current python version (where f2b-modules located/installed)
+updatePyExec(os.path.dirname(__file__))
+
 def get_opt_parser():
 	# use module docstring for help output
 	p = OptionParser(
diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index fe4b6591..68a2cb5b 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env fail2ban-python
 # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
 #
 # Written in Python to reuse built-in Python batteries and not depend on
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 7660e64a..99210b17 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -118,6 +118,24 @@ class FormatterWithTraceBack(logging.Formatter):
 		return logging.Formatter.format(self, record)
 
 
+def updatePyExec(bindir, executable=None):
+	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
+	"""
+	bindir = os.path.realpath(bindir)
+	if executable is None:
+		executable = sys.executable
+	pypath = os.path.join(bindir, 'fail2ban-python')
+	# if not exists or point to another version - update link:
+	isfile = os.path.isfile(pypath)
+	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
+		if isfile:
+			os.unlink(pypath)
+		os.symlink(executable, pypath)
+	# extend current environment path (e.g. if fail2ban not yet installed):
+	if bindir not in os.environ["PATH"].split(os.pathsep):
+		os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir;
+
+
 def getLogger(name):
 	"""Get logging.Logger instance with Fail2Ban logger name convention
 	"""
diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py
index f4fcfcb9..03588594 100755
--- a/fail2ban/tests/files/config/apache-auth/digest.py
+++ b/fail2ban/tests/files/config/apache-auth/digest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env fail2ban-python
 import requests
 
 try:
diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py
index 473980ec..7011b51b 100755
--- a/fail2ban/tests/files/ignorecommand.py
+++ b/fail2ban/tests/files/ignorecommand.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env fail2ban-python
 import sys
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index df347e77..b910d94c 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -73,7 +73,7 @@ class HelpersTest(unittest.TestCase):
 
 def _getSysPythonVersion():
 	import subprocess, locale
-	sysVerCmd = "python -c 'import sys; print(tuple(sys.version_info))'"
+	sysVerCmd = "fail2ban-python -c 'import sys; print(tuple(sys.version_info))'"
 	if sys.version_info >= (2,7):
 		sysVer = subprocess.check_output(sysVerCmd, shell=True)
 	else:
@@ -144,6 +144,10 @@ class SetupTest(unittest.TestCase):
 					  'etc/fail2ban/jail.conf'):
 				self.assertTrue(os.path.exists(os.path.join(tmp, f)),
 								msg="Can't find %s" % f)
+			self.assertEqual(
+				os.path.realpath(os.path.join(tmp, 'usr/local/bin/fail2ban-python')),
+				os.path.realpath(sys.executable))
+
 		finally:
 			# clean up
 			shutil.rmtree(tmp)
diff --git a/setup.py b/setup.py
index 5579c981..fa66d4d6 100755
--- a/setup.py
+++ b/setup.py
@@ -38,12 +38,46 @@ except ImportError:
 	# python 2.x
 	from distutils.command.build_py import build_py
 	from distutils.command.build_scripts import build_scripts
+# all versions
+from distutils.command.install_scripts import install_scripts
+
 import os
 from os.path import isfile, join, isdir, realpath
 import sys
 import warnings
 from glob import glob
 
+
+def updatePyExec(bindir, executable=None):
+	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
+	"""
+	bindir = os.path.realpath(bindir)
+	if executable is None:
+		executable = sys.executable
+	pypath = os.path.join(bindir, 'fail2ban-python')
+	# if not exists or point to another version - update link:
+	isfile = os.path.isfile(pypath)
+	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
+		if isfile:
+			os.unlink(pypath)
+		os.symlink(executable, pypath)
+
+
+# Wrapper to install python binding (to current python version):
+class install_scripts_f2b(install_scripts):
+
+	def get_outputs(self):
+		outputs = install_scripts.get_outputs(self)
+		fn = None
+		for fn in outputs:
+			if os.path.basename(fn) == 'fail2ban-server':
+				break
+		bindir = os.path.dirname(fn)
+		print('creating fail2ban-python binding -> %s' % (bindir,))
+		updatePyExec(bindir)
+		return outputs
+
+
 if setuptools and "test" in sys.argv:
 	import logging
 	logSys = logging.getLogger("fail2ban")
@@ -99,12 +133,16 @@ setup(
 	url = "http://www.fail2ban.org",
 	license = "GPL",
 	platforms = "Posix",
-	cmdclass = {'build_py': build_py, 'build_scripts': build_scripts},
+	cmdclass = {
+		'build_py': build_py, 'build_scripts': build_scripts, 
+		'install_scripts': install_scripts_f2b
+	},
 	scripts = [
 		'bin/fail2ban-client',
 		'bin/fail2ban-server',
 		'bin/fail2ban-regex',
 		'bin/fail2ban-testcases',
+		# 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper
 	],
 	packages = [
 		'fail2ban',

From 08af8de98171ad37a190dfaafccd629c83afe1a1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Aug 2016 19:57:40 +0200
Subject: [PATCH 141/623] compatibility fix (virtualenv, running test cases in
 py3)

---
 MANIFEST                       |  1 +
 bin/fail2ban-testcases         |  3 ++-
 fail2ban/helpers.py            | 18 ---------------
 fail2ban/setup.py              | 42 ++++++++++++++++++++++++++++++++++
 fail2ban/tests/misctestcase.py | 33 ++++++++++++++++----------
 setup.py                       | 18 ++++-----------
 6 files changed, 70 insertions(+), 45 deletions(-)
 create mode 100644 fail2ban/setup.py

diff --git a/MANIFEST b/MANIFEST
index 36110c83..512b1d03 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -200,6 +200,7 @@ fail2ban/server/strptime.py
 fail2ban/server/ticket.py
 fail2ban/server/transmitter.py
 fail2ban/server/utils.py
+fail2ban/setup.py
 fail2ban-testcases-all
 fail2ban-testcases-all-python3
 fail2ban/tests/action_d/__init__.py
diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index dcc55421..34d392e4 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -38,7 +38,8 @@ if os.path.exists("fail2ban/__init__.py"):
 from fail2ban.version import version
 
 from fail2ban.tests.utils import gatherTests
-from fail2ban.helpers import updatePyExec, FormatterWithTraceBack, getLogger
+from fail2ban.helpers import FormatterWithTraceBack, getLogger
+from fail2ban.setup import updatePyExec
 from fail2ban.server.mytime import MyTime
 
 from optparse import OptionParser, Option
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 99210b17..7660e64a 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -118,24 +118,6 @@ class FormatterWithTraceBack(logging.Formatter):
 		return logging.Formatter.format(self, record)
 
 
-def updatePyExec(bindir, executable=None):
-	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
-	"""
-	bindir = os.path.realpath(bindir)
-	if executable is None:
-		executable = sys.executable
-	pypath = os.path.join(bindir, 'fail2ban-python')
-	# if not exists or point to another version - update link:
-	isfile = os.path.isfile(pypath)
-	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
-		if isfile:
-			os.unlink(pypath)
-		os.symlink(executable, pypath)
-	# extend current environment path (e.g. if fail2ban not yet installed):
-	if bindir not in os.environ["PATH"].split(os.pathsep):
-		os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir;
-
-
 def getLogger(name):
 	"""Get logging.Logger instance with Fail2Ban logger name convention
 	"""
diff --git a/fail2ban/setup.py b/fail2ban/setup.py
new file mode 100644
index 00000000..87d50e66
--- /dev/null
+++ b/fail2ban/setup.py
@@ -0,0 +1,42 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Serg G. Brester"
+__license__ = "GPL"
+
+import os
+import sys
+
+
+def updatePyExec(bindir, executable=None):
+	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
+	"""
+	bindir = os.path.realpath(bindir)
+	if executable is None:
+		executable = sys.executable
+	pypath = os.path.join(bindir, 'fail2ban-python')
+	# if not exists or point to another version - update link:
+	isfile = os.path.isfile(pypath)
+	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
+		if isfile:
+			os.unlink(pypath)
+		os.symlink(executable, pypath)
+	# extend current environment path (e.g. if fail2ban not yet installed):
+	if bindir not in os.environ["PATH"].split(os.pathsep):
+		os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir;
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index b910d94c..7028eeb1 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -71,16 +71,21 @@ class HelpersTest(unittest.TestCase):
 		self.assertEqual(splitwords(' 1\n  2, 3'), ['1', '2', '3'])
 
 
+if sys.version_info >= (2,7):
+	def _sh_call(cmd):
+		import subprocess, locale
+		ret = subprocess.check_output(cmd, shell=True)
+		if sys.version_info >= (3,):
+			ret = ret.decode(locale.getpreferredencoding(), 'replace')
+		return str(ret).rstrip()
+else:
+	def _sh_call(cmd):
+		import subprocess
+		ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
+		return str(ret).rstrip()
+
 def _getSysPythonVersion():
-	import subprocess, locale
-	sysVerCmd = "fail2ban-python -c 'import sys; print(tuple(sys.version_info))'"
-	if sys.version_info >= (2,7):
-		sysVer = subprocess.check_output(sysVerCmd, shell=True)
-	else:
-		sysVer = subprocess.Popen(sysVerCmd, shell=True, stdout=subprocess.PIPE).stdout.read()
-	if sys.version_info >= (3,):
-		sysVer = sysVer.decode(locale.getpreferredencoding(), 'replace')
-	return str(sysVer).rstrip()
+	return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'")
 
 class SetupTest(unittest.TestCase):
 
@@ -144,9 +149,13 @@ class SetupTest(unittest.TestCase):
 					  'etc/fail2ban/jail.conf'):
 				self.assertTrue(os.path.exists(os.path.join(tmp, f)),
 								msg="Can't find %s" % f)
-			self.assertEqual(
-				os.path.realpath(os.path.join(tmp, 'usr/local/bin/fail2ban-python')),
-				os.path.realpath(sys.executable))
+			# Because the install (test) path in virtual-env differs from some development-env,
+			# it is not a `tmp + '/usr/local/bin/'`, so search for it:
+			installedPath = _sh_call('find ' + tmp+ ' -name fail2ban-python').split('\n')
+			self.assertTrue(len(installedPath) > 0)
+			for installedPath in installedPath:
+				self.assertEqual(
+					os.path.realpath(installedPath), os.path.realpath(sys.executable))
 
 		finally:
 			# clean up
diff --git a/setup.py b/setup.py
index fa66d4d6..be1fe685 100755
--- a/setup.py
+++ b/setup.py
@@ -47,20 +47,7 @@ import sys
 import warnings
 from glob import glob
 
-
-def updatePyExec(bindir, executable=None):
-	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
-	"""
-	bindir = os.path.realpath(bindir)
-	if executable is None:
-		executable = sys.executable
-	pypath = os.path.join(bindir, 'fail2ban-python')
-	# if not exists or point to another version - update link:
-	isfile = os.path.isfile(pypath)
-	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
-		if isfile:
-			os.unlink(pypath)
-		os.symlink(executable, pypath)
+from fail2ban.setup import updatePyExec
 
 
 # Wrapper to install python binding (to current python version):
@@ -78,6 +65,9 @@ class install_scripts_f2b(install_scripts):
 		return outputs
 
 
+# Update fail2ban-python env to current python version (where f2b-modules located/installed)
+updatePyExec(os.path.join(os.path.dirname(__file__), 'bin'))
+
 if setuptools and "test" in sys.argv:
 	import logging
 	logSys = logging.getLogger("fail2ban")

From f390a82b597a7c6aa39fff33b6ef106ed0b45bdb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Aug 2016 21:37:16 +0200
Subject: [PATCH 142/623] python 2.6 compatibility minor fix - no skip +
 BaseException.message seems to be deprecated

---
 fail2ban/tests/utils.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 2838fa05..f6333fb8 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -104,9 +104,10 @@ def initTests(opts):
 		# (prevent long sleeping during test cases ... less time goes to sleep):
 		Utils.DEFAULT_SLEEP_TIME = 0.0025
 		Utils.DEFAULT_SLEEP_INTERVAL = 0.0005
-		def F2B_SkipIfFast():
-			raise unittest.SkipTest('Skip test because of "--fast"')
-		unittest.F2B.SkipIfFast = F2B_SkipIfFast
+		if sys.version_info >= (2,7): # no skip in previous version:
+			def F2B_SkipIfFast():
+				raise unittest.SkipTest('Skip test because of "--fast"')
+			unittest.F2B.SkipIfFast = F2B_SkipIfFast
 	else:
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
@@ -333,8 +334,8 @@ if not hasattr(unittest.TestCase, 'assertRaisesRegexp'):
 		try:
 			fun(*args, **kwargs)
 		except exccls as e:
-			if re.search(regexp, e.message) is None:
-				self.fail('\"%s\" does not match \"%s\"' % (regexp, e.message))
+			if re.search(regexp, str(e)) is None:
+				self.fail('\"%s\" does not match \"%s\"' % (regexp, e))
 		else:
 			self.fail('%s not raised' % getattr(exccls, '__name__'))
 	unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp

From c8e7c1f7f4527da431b41bd7d23d1b51dd135b20 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 16:41:55 +0200
Subject: [PATCH 143/623] BF: prefer sys.argv[0] by retrieving of root resp.
 bin path: __file__ seems to be overwritten sometimes on some python versions
 (e.g. bug of 2.6 by running under cProfile, etc.)

---
 bin/fail2ban-testcases | 6 +++++-
 setup.py               | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 34d392e4..7e5f7aca 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -45,7 +45,11 @@ from fail2ban.server.mytime import MyTime
 from optparse import OptionParser, Option
 
 # Update fail2ban-python env to current python version (where f2b-modules located/installed)
-updatePyExec(os.path.dirname(__file__))
+bindir = os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'fail2ban-testcases' else __file__
+)
+updatePyExec(bindir)
 
 def get_opt_parser():
 	# use module docstring for help output
diff --git a/setup.py b/setup.py
index be1fe685..11d0a20b 100755
--- a/setup.py
+++ b/setup.py
@@ -66,7 +66,11 @@ class install_scripts_f2b(install_scripts):
 
 
 # Update fail2ban-python env to current python version (where f2b-modules located/installed)
-updatePyExec(os.path.join(os.path.dirname(__file__), 'bin'))
+rootdir = os.path.realpath(os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__
+))
+updatePyExec(os.path.join(rootdir, 'bin'))
 
 if setuptools and "test" in sys.argv:
 	import logging

From db24edba818a323e78d3d6875360a72e2a5bb345 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 12 Aug 2016 17:29:42 +0200
Subject: [PATCH 144/623] Update README.md

show status of builds (travis and coverage) from default branch (0.10)
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index bedbb2ec..7d541e79 100644
--- a/README.md
+++ b/README.md
@@ -73,11 +73,11 @@ fail2ban(1) and jail.conf(5)  manpages for further references.
 Code status:
 ------------
 
-* [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=master)](https://travis-ci.org/fail2ban/fail2ban) travis-ci.org (master branch)
+* [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) travis-ci.org (0.10 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=master)](https://travis-ci.org/fail2ban/fail2ban) travis-ci.org (master branch)
 
-* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=master)](https://coveralls.io/r/fail2ban/fail2ban)
+* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=0.10)](https://coveralls.io/r/fail2ban/fail2ban?https://coveralls.io/github/fail2ban/fail2ban?branch=0.10)
 
-* [![codecov.io](https://codecov.io/github/fail2ban/fail2ban/coverage.svg?branch=master)](https://codecov.io/github/fail2ban/fail2ban?branch=master)
+* [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10)
 
 Contact:
 --------

From 461e9bab56027e257a836132e753a044bfa93cc2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 11:07:31 +0200
Subject: [PATCH 145/623] better handling of setup in test-cases (prevent
 execution of 2to3 twice, full logging in heavydebug, etc.)

---
 bin/fail2ban-testcases         |  3 ++-
 fail2ban/tests/misctestcase.py | 11 ++++++----
 setup.py                       | 39 +++++++++++++++++++++++++++-------
 3 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 7e5f7aca..76a8e6ec 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -113,6 +113,7 @@ else: # pragma: no cover
 	# ticking, unless like with '-l critical' which would be silent
 	# unless error occurs
 	logSys.setLevel(getattr(logging, 'CRITICAL'))
+opts.log_level = logSys.level
 
 # Add the default logging handler
 stdout = logging.StreamHandler(sys.stdout)
@@ -140,7 +141,7 @@ logSys.addHandler(stdout)
 #
 # Let know the version
 #
-if not opts.log_level or opts.log_level != 'critical': # pragma: no cover
+if opts.log_level != logging.CRITICAL: # pragma: no cover
 	print("Fail2ban %s test suite. Python %s. Please wait..." \
 			% (version, str(sys.version).replace('\n', '')))
 
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 7028eeb1..7558851a 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -87,6 +87,7 @@ else:
 def _getSysPythonVersion():
 	return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'")
 
+
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
@@ -108,9 +109,11 @@ class SetupTest(unittest.TestCase):
 		if not self.setup:
 			return			  # if verbose skip didn't work out
 		tmp = tempfile.mkdtemp()
+		# suppress stdout (and stderr) if not heavydebug
+		supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
-			os.system("%s %s install --root=%s >/dev/null"
-					  % (sys.executable, self.setup, tmp))
+			os.system("%s %s install --disable-2to3 --dry-run --root=%s%s"
+					  % (sys.executable, self.setup, tmp, supdbgout))
 
 			def strippath(l):
 				return [x[len(tmp)+1:] for x in l]
@@ -161,8 +164,8 @@ class SetupTest(unittest.TestCase):
 			# clean up
 			shutil.rmtree(tmp)
 			# remove build directory
-			os.system("%s %s clean --all >/dev/null 2>&1"
-					  % (sys.executable, self.setup))
+			os.system("%s %s clean --all%s"
+					  % (sys.executable, self.setup, (supdbgout + ' 2>&1') if supdbgout else ''))
 
 
 class TestsUtilsTest(LogCaptureTestCase):
diff --git a/setup.py b/setup.py
index f9f57976..fbffbaf8 100755
--- a/setup.py
+++ b/setup.py
@@ -31,17 +31,19 @@ except ImportError:
 	setuptools = None
 	from distutils.core import setup
 
+# all versions
+from distutils.command.build_py import build_py
+from distutils.command.build_scripts import build_scripts
+from distutils.command.install_scripts import install_scripts
+from distutils.command.install import install
 try:
 	# python 3.x
-	from distutils.command.build_py import build_py_2to3 as build_py
-	from distutils.command.build_scripts \
-		import build_scripts_2to3 as build_scripts
+	from distutils.command.build_py import build_py_2to3
+	from distutils.command.build_scripts import build_scripts_2to3
+	_2to3 = True
 except ImportError:
 	# python 2.x
-	from distutils.command.build_py import build_py
-	from distutils.command.build_scripts import build_scripts
-# all versions
-from distutils.command.install_scripts import install_scripts
+	_2to3 = False
 
 import os
 from os.path import isfile, join, isdir, realpath
@@ -66,6 +68,27 @@ class install_scripts_f2b(install_scripts):
 		updatePyExec(bindir)
 		return outputs
 
+# Wrapper to specify fail2ban own options:
+class install_command_f2b(install):
+	user_options = install.user_options + [
+		('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'),
+	]
+	def initialize_options(self):
+		self.disable_2to3 = None
+		install.initialize_options(self)
+	def finalize_options(self):
+		global _2to3
+		## in the test cases 2to3 should be already done (fail2ban-2to3):
+		if self.disable_2to3:
+			_2to3 = False
+		if _2to3:
+			cmdclass = self.distribution.cmdclass
+			cmdclass['build_py'] = build_py_2to3
+			cmdclass['build_scripts'] = build_scripts_2to3
+		install.finalize_options(self)
+	def run(self):
+		install.run(self)
+
 
 # Update fail2ban-python env to current python version (where f2b-modules located/installed)
 rootdir = os.path.realpath(os.path.dirname(
@@ -143,7 +166,7 @@ setup(
 	platforms = "Posix",
 	cmdclass = {
 		'build_py': build_py, 'build_scripts': build_scripts, 
-		'install_scripts': install_scripts_f2b
+		'install_scripts': install_scripts_f2b, 'install': install_command_f2b
 	},
 	scripts = [
 		'bin/fail2ban-client',

From b2ee3b120e6629ff65dcc1c4523db918b514131e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 16:39:40 +0200
Subject: [PATCH 146/623] optimized logging using LogCaptureTestCase (new
 faster handler _MemHandler); backwards compatibility for python 2.6 (SkipTest
 implemented), so options --fast and --no-network may be used now within
 python2.6 also;

---
 fail2ban/tests/utils.py | 64 +++++++++++++++++++++++++++++++++++------
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index f6333fb8..3f0822dd 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -33,7 +33,7 @@ import sys
 import time
 import unittest
 
-from StringIO import StringIO
+from cStringIO import StringIO
 from functools import wraps
 
 from ..helpers import getLogger
@@ -96,6 +96,23 @@ def with_tmpdir(f):
 	return wrapper
 
 
+# backwards compatibility to python 2.6:
+if not hasattr(unittest, 'SkipTest'): # pragma: no cover
+	class SkipTest(Exception):
+		pass
+	unittest.SkipTest = SkipTest
+	_org_AddError = unittest._TextTestResult.addError
+	def addError(self, test, err):
+		if err[0] is SkipTest: 
+			if self.showAll:
+				self.stream.writeln(str(err[1]))
+			elif self.dots:
+				self.stream.write('s')
+				self.stream.flush()
+			return
+		_org_AddError(self, test, err)
+	unittest._TextTestResult.addError = addError
+
 def initTests(opts):
 	unittest.F2B = F2B(opts)
 	# --fast :
@@ -104,10 +121,9 @@ def initTests(opts):
 		# (prevent long sleeping during test cases ... less time goes to sleep):
 		Utils.DEFAULT_SLEEP_TIME = 0.0025
 		Utils.DEFAULT_SLEEP_INTERVAL = 0.0005
-		if sys.version_info >= (2,7): # no skip in previous version:
-			def F2B_SkipIfFast():
-				raise unittest.SkipTest('Skip test because of "--fast"')
-			unittest.F2B.SkipIfFast = F2B_SkipIfFast
+		def F2B_SkipIfFast():
+			raise unittest.SkipTest('Skip test because of "--fast"')
+		unittest.F2B.SkipIfFast = F2B_SkipIfFast
 	else:
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
@@ -368,6 +384,38 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 
 class LogCaptureTestCase(unittest.TestCase):
 
+	class _MemHandler(logging.StreamHandler):
+		def __init__(self):
+			self._recs = list()
+			self._strm = StringIO()
+			logging.StreamHandler.__init__(self, self._strm)
+			# 
+		def truncate(self, size=None):
+			"""Truncate the internal buffer and records."""
+			if size:
+				raise Exception('invalid size argument: %r, should be None or 0' % size)
+			self._strm.truncate(0)
+			self._val = None
+			self._recs = list()
+			# 
+		def getvalue(self):
+			"""Return current buffer as whole string."""
+			# cached:
+			if self._val is not None:
+				return self._val
+			# submit already emitted (delivered to handle) records:
+			for record in self._recs:
+				logging.StreamHandler.handle(self, record)
+			self._recs = list()
+			# cache and return:
+			self._val = self._strm.getvalue()
+			return self._val
+			# 
+		def handle(self, record):
+			"""Handle the specified record."""
+			self._val = None
+			self._recs.append(record)
+
 	def setUp(self):
 
 		# For extended testing of what gets output into logging
@@ -378,13 +426,13 @@ class LogCaptureTestCase(unittest.TestCase):
 		self._old_level = logSys.level
 		self._old_handlers = logSys.handlers
 		# Let's log everything into a string
-		self._log = StringIO()
-		logSys.handlers = [logging.StreamHandler(self._log)]
+		self._log = LogCaptureTestCase._MemHandler()
+		logSys.handlers = [self._log]
 		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
 			print("")
 			logSys.handlers += self._old_handlers
 			logSys.debug('='*10 + ' %s ' + '='*20, self.id())
-		logSys.setLevel(getattr(logging, 'DEBUG'))
+		logSys.setLevel(logging.DEBUG)
 
 	def tearDown(self):
 		"""Call after every test case."""

From fcae05190f0628417623df26f1dbfd82c11bcc5a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 20:54:20 +0200
Subject: [PATCH 147/623] move common initializing functionality from
 fail2ban-testcases to tests/utils.py, to afford setting of default options by
 test execution from setup process.

---
 bin/fail2ban-testcases  | 52 +++--------------------------
 fail2ban/tests/utils.py | 74 ++++++++++++++++++++++++++++++++++++++---
 2 files changed, 73 insertions(+), 53 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 76a8e6ec..dbbcf3f2 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -37,10 +37,8 @@ if os.path.exists("fail2ban/__init__.py"):
 	sys.path.insert(0, ".")
 from fail2ban.version import version
 
-from fail2ban.tests.utils import gatherTests
-from fail2ban.helpers import FormatterWithTraceBack, getLogger
+from fail2ban.tests.utils import initProcess, gatherTests
 from fail2ban.setup import updatePyExec
-from fail2ban.server.mytime import MyTime
 
 from optparse import OptionParser, Option
 
@@ -91,52 +89,10 @@ parser = get_opt_parser()
 (opts, regexps) = parser.parse_args()
 
 #
-# Logging
+# Process initialization corresponding options (logging, default options, etc.)
 #
-logSys = getLogger("fail2ban")
-
-# Numerical level of verbosity corresponding to a log "level"
-verbosity = {'heavydebug': 4,
-			 'debug': 3,
-			 'info': 2,
-			 'notice': 2,
-			 'warning': 1,
-			 'error': 1,
-			 'critical': 0,
-			 None: 1}[opts.log_level]
-
-if opts.log_level is not None: # pragma: no cover
-	# so we had explicit settings
-	logSys.setLevel(getattr(logging, opts.log_level.upper()))
-else: # pragma: no cover
-	# suppress the logging but it would leave unittests' progress dots
-	# ticking, unless like with '-l critical' which would be silent
-	# unless error occurs
-	logSys.setLevel(getattr(logging, 'CRITICAL'))
-opts.log_level = logSys.level
-
-# Add the default logging handler
-stdout = logging.StreamHandler(sys.stdout)
-
-fmt = ' %(message)s'
-
-if opts.log_traceback:
-	Formatter = FormatterWithTraceBack
-	fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt
-else:
-	Formatter = logging.Formatter
-
-# Custom log format for the verbose tests runs
-if verbosity > 1: # pragma: no cover
-	if verbosity > 3:
-		fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
-	if verbosity > 2:
-		fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
-	else:
-		fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
-#
-stdout.setFormatter(Formatter(fmt))
-logSys.addHandler(stdout)
+opts = initProcess(opts)
+verbosity = opts.verbosity
 
 #
 # Let know the version
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 3f0822dd..b0dc646f 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -59,12 +59,73 @@ if not CONFIG_DIR:
 os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
 	os.path.abspath(__file__)))))
 
-class F2B(optparse.Values):
-	def __init__(self, opts={}):
-		self.__dict__ = opts.__dict__ if opts else {
-			'fast': False, 'memory_db':False, 'no_gamin': False, 'no_network': False, 
-			"negate_re": False,
+# Default options, if running from installer (setup.py):
+class DefaultTestOptions(optparse.Values):
+	def __init__(self):
+		self.__dict__ = {
+			'log_level': None, 'log_traceback': None, 'full_traceback': None,
+			'fast': False, 'memory_db': False, 'no_gamin': False,
+			'no_network': False, 'negate_re': False
 		}
+
+#
+# Initialization
+#
+def initProcess(opts):
+	# Logger:
+	logSys = getLogger("fail2ban")
+
+	# Numerical level of verbosity corresponding to a log "level"
+	verbosity = {'heavydebug': 4,
+				 'debug': 3,
+				 'info': 2,
+				 'notice': 2,
+				 'warning': 1,
+				 'error': 1,
+				 'critical': 0,
+				 None: 1}[opts.log_level]
+	opts.verbosity = verbosity
+
+	if opts.log_level is not None: # pragma: no cover
+		# so we had explicit settings
+		logSys.setLevel(getattr(logging, opts.log_level.upper()))
+	else: # pragma: no cover
+		# suppress the logging but it would leave unittests' progress dots
+		# ticking, unless like with '-l critical' which would be silent
+		# unless error occurs
+		logSys.setLevel(logging.CRITICAL)
+	opts.log_level = logSys.level
+
+	# Add the default logging handler
+	stdout = logging.StreamHandler(sys.stdout)
+
+	fmt = ' %(message)s'
+
+	if opts.log_traceback: # pragma: no cover
+		from ..helpers import FormatterWithTraceBack as Formatter
+		fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt
+	else:
+		Formatter = logging.Formatter
+
+	# Custom log format for the verbose tests runs
+	if verbosity > 1: # pragma: no cover
+		if verbosity > 3:
+			fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
+		if verbosity > 2:
+			fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
+		else:
+			fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
+
+	#
+	stdout.setFormatter(Formatter(fmt))
+	logSys.addHandler(stdout)
+	#
+	return opts;
+
+
+class F2B(optparse.Values):
+	def __init__(self, opts):
+		self.__dict__ = opts.__dict__
 		if self.fast:
 			self.memory_db = True
 			self.no_gamin = True
@@ -114,6 +175,9 @@ if not hasattr(unittest, 'SkipTest'): # pragma: no cover
 	unittest._TextTestResult.addError = addError
 
 def initTests(opts):
+	## if running from installer (setup.py):
+	if not opts:
+		opts = initProcess(DefaultTestOptions())
 	unittest.F2B = F2B(opts)
 	# --fast :
 	if unittest.F2B.fast: # pragma: no cover

From 2e530fcf72c9e9bae321b37b625728bf9f88f71b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 21:26:26 +0200
Subject: [PATCH 148/623] setuptools compatibility fix

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index fbffbaf8..1efb1da7 100755
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,7 @@ import platform
 try:
 	import setuptools
 	from setuptools import setup
+	from setuptools.command.install import install
 except ImportError:
 	setuptools = None
 	from distutils.core import setup
@@ -35,7 +36,8 @@ except ImportError:
 from distutils.command.build_py import build_py
 from distutils.command.build_scripts import build_scripts
 from distutils.command.install_scripts import install_scripts
-from distutils.command.install import install
+if setuptools is None:
+	from distutils.command.install import install
 try:
 	# python 3.x
 	from distutils.command.build_py import build_py_2to3

From eb72660eeec7fc1aea4483450c37fc1c601c7653 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 21:32:18 +0200
Subject: [PATCH 149/623] easy-fix: reset fail2ban link to python, if it
 targeted unavailable file

---
 fail2ban/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/setup.py b/fail2ban/setup.py
index 87d50e66..efaa31a4 100644
--- a/fail2ban/setup.py
+++ b/fail2ban/setup.py
@@ -32,7 +32,7 @@ def updatePyExec(bindir, executable=None):
 		executable = sys.executable
 	pypath = os.path.join(bindir, 'fail2ban-python')
 	# if not exists or point to another version - update link:
-	isfile = os.path.isfile(pypath)
+	isfile = os.path.isfile(os.path.realpath(pypath))
 	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
 		if isfile:
 			os.unlink(pypath)

From 83029b89b9121a153f28b274975837def9a18c39 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 Aug 2016 21:47:31 +0200
Subject: [PATCH 150/623] amend to 2e530fcf72c9e9bae321b37b625728bf9f88f71b:
 setuptools compatibility fix - just to be sure if install_scripts will be
 incompatible tomorrow

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 1efb1da7..b3645f01 100755
--- a/setup.py
+++ b/setup.py
@@ -28,6 +28,7 @@ try:
 	import setuptools
 	from setuptools import setup
 	from setuptools.command.install import install
+	from setuptools.command.install_scripts import install_scripts
 except ImportError:
 	setuptools = None
 	from distutils.core import setup
@@ -35,9 +36,9 @@ except ImportError:
 # all versions
 from distutils.command.build_py import build_py
 from distutils.command.build_scripts import build_scripts
-from distutils.command.install_scripts import install_scripts
 if setuptools is None:
 	from distutils.command.install import install
+	from distutils.command.install_scripts import install_scripts
 try:
 	# python 3.x
 	from distutils.command.build_py import build_py_2to3

From 6fcf8192f3b67ac7c0789205df35f4596dac5807 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Aug 2016 13:51:50 +0200
Subject: [PATCH 151/623] code review

---
 fail2ban/tests/utils.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index b0dc646f..1e33ad88 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -449,11 +449,19 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 class LogCaptureTestCase(unittest.TestCase):
 
 	class _MemHandler(logging.StreamHandler):
+		"""Logging handler helper
+		
+		Affords not to delegate logging to StreamHandler at all,
+		format lazily on demand in getvalue.
+		Increases performance inside the LogCaptureTestCase tests, because there
+		the log level set to DEBUG.
+		"""
+
 		def __init__(self):
 			self._recs = list()
 			self._strm = StringIO()
 			logging.StreamHandler.__init__(self, self._strm)
-			# 
+			
 		def truncate(self, size=None):
 			"""Truncate the internal buffer and records."""
 			if size:
@@ -461,7 +469,7 @@ class LogCaptureTestCase(unittest.TestCase):
 			self._strm.truncate(0)
 			self._val = None
 			self._recs = list()
-			# 
+			 
 		def getvalue(self):
 			"""Return current buffer as whole string."""
 			# cached:
@@ -474,7 +482,7 @@ class LogCaptureTestCase(unittest.TestCase):
 			# cache and return:
 			self._val = self._strm.getvalue()
 			return self._val
-			# 
+			 
 		def handle(self, record):
 			"""Handle the specified record."""
 			self._val = None

From d75af2f9008b7733b883c4596ba3eaa05f04c7e3 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 15 Aug 2016 13:52:32 +0200
Subject: [PATCH 152/623] travis: coverage python 3.5 (#1511)

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 9ef607da..8aa781be 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ python:
   # - 3.2
   - 3.3
   - 3.4
+  - 3.5
   - pypy3
 before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi

From 2b5c5103f463e1fa6ab646a28d813af63389a38f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Aug 2016 21:54:33 +0200
Subject: [PATCH 153/623] safer multi-threaded execution inside (logging with
 lock inside LogCaptureTestCase logging handler); additionally developer
 option `--log-direct` for fail2ban-testcases: disable lazy logging inside
 LogCaptureTestCase (default log-lazy is on)

---
 bin/fail2ban-testcases         |  4 ++
 fail2ban/tests/misctestcase.py | 11 ++++++
 fail2ban/tests/utils.py        | 68 ++++++++++++++++++++++------------
 3 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index dbbcf3f2..4a93ac95 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -61,6 +61,10 @@ def get_opt_parser():
 			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
+		Option("--log-direct", action="store_false",
+			   dest="log_lazy",
+			   default=True,
+			   help="Prevent lazy logging inside tests"),
 		Option('-n', "--no-network", action="store_true",
 			   dest="no_network",
 			   help="Do not run tests that require the network"),
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 7558851a..83e05713 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -306,6 +306,17 @@ class TestsUtilsTest(LogCaptureTestCase):
 		self.assertTrue(pindex > 10)	  # we should have some traceback
 		self.assertEqual(s[:pindex], s[pindex+1:pindex*2 + 1])
 
+	def testLazyLogging(self):
+		logSys = DefLogSys
+		if unittest.F2B.log_lazy:
+			# wrong logging syntax will throw an error lazy (on demand):
+			logSys.debug('test', 1, 2, 3)
+			self.assertRaisesRegexp(Exception, 'not all arguments converted', lambda: self.assertNotLogged('test'))
+		else: # pragma: no cover
+			# wrong logging syntax will throw an error directly:
+			self.assertRaisesRegexp(Exception, 'not all arguments converted', lambda: logSys.debug('test', 1, 2, 3))
+
+
 iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
 
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 1e33ad88..232da780 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -31,6 +31,7 @@ import tempfile
 import shutil
 import sys
 import time
+import threading
 import unittest
 
 from cStringIO import StringIO
@@ -63,7 +64,8 @@ os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
 class DefaultTestOptions(optparse.Values):
 	def __init__(self):
 		self.__dict__ = {
-			'log_level': None, 'log_traceback': None, 'full_traceback': None,
+			'log_level': None, 'log_lazy': True, 
+			'log_traceback': None, 'full_traceback': None,
 			'fast': False, 'memory_db': False, 'no_gamin': False,
 			'no_network': False, 'negate_re': False
 		}
@@ -123,7 +125,7 @@ def initProcess(opts):
 	return opts;
 
 
-class F2B(optparse.Values):
+class F2B(DefaultTestOptions):
 	def __init__(self, opts):
 		self.__dict__ = opts.__dict__
 		if self.fast:
@@ -448,7 +450,7 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 
 class LogCaptureTestCase(unittest.TestCase):
 
-	class _MemHandler(logging.StreamHandler):
+	class _MemHandler(logging.Handler):
 		"""Logging handler helper
 		
 		Affords not to delegate logging to StreamHandler at all,
@@ -457,36 +459,56 @@ class LogCaptureTestCase(unittest.TestCase):
 		the log level set to DEBUG.
 		"""
 
-		def __init__(self):
+		def __init__(self, lazy=True):
+			self._lock = threading.Lock()
+			self._val = None
 			self._recs = list()
 			self._strm = StringIO()
-			logging.StreamHandler.__init__(self, self._strm)
+			logging.Handler.__init__(self)
+			if lazy:
+				self.handle = self._handle_lazy
 			
 		def truncate(self, size=None):
 			"""Truncate the internal buffer and records."""
 			if size:
 				raise Exception('invalid size argument: %r, should be None or 0' % size)
-			self._strm.truncate(0)
-			self._val = None
-			self._recs = list()
-			 
+			with self._lock:
+				self._strm.truncate(0)
+				self._val = None
+				self._recs = list()
+
+		def __write(self, record):
+			msg = record.getMessage() + '\n'
+			try:
+				self._strm.write(msg)
+			except UnicodeEncodeError:
+				self._strm.write(msg.encode('UTF-8'))
+
 		def getvalue(self):
 			"""Return current buffer as whole string."""
-			# cached:
-			if self._val is not None:
+			with self._lock:
+				# cached:
+				if self._val is not None:
+					return self._val
+				# submit already emitted (delivered to handle) records:
+				for record in self._recs:
+					self.__write(record)
+				self._recs = list()
+				# cache and return:
+				self._val = self._strm.getvalue()
 				return self._val
-			# submit already emitted (delivered to handle) records:
-			for record in self._recs:
-				logging.StreamHandler.handle(self, record)
-			self._recs = list()
-			# cache and return:
-			self._val = self._strm.getvalue()
-			return self._val
 			 
-		def handle(self, record):
-			"""Handle the specified record."""
-			self._val = None
-			self._recs.append(record)
+		def handle(self, record): # pragma: no cover
+			"""Handle the specified record direct (not lazy)"""
+			with self._lock:
+				self._val = None
+				self.__write(record)
+
+		def _handle_lazy(self, record):
+			"""Lazy handle the specified record on demand"""
+			with self._lock:
+				self._val = None
+				self._recs.append(record)
 
 	def setUp(self):
 
@@ -498,7 +520,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		self._old_level = logSys.level
 		self._old_handlers = logSys.handlers
 		# Let's log everything into a string
-		self._log = LogCaptureTestCase._MemHandler()
+		self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy)
 		logSys.handlers = [self._log]
 		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
 			print("")

From e3f4ea7284113be25ac2040487ffd7ba9d5604e3 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 16 Aug 2016 10:43:01 +0200
Subject: [PATCH 154/623] Update README.md

fixed coveralls url to the default branch
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7d541e79..be0be636 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ Code status:
 
 * [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) travis-ci.org (0.10 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=master)](https://travis-ci.org/fail2ban/fail2ban) travis-ci.org (master branch)
 
-* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=0.10)](https://coveralls.io/r/fail2ban/fail2ban?https://coveralls.io/github/fail2ban/fail2ban?branch=0.10)
+* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10)
 
 * [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10)
 

From 7b7d60c486633410c09bf7a7ec68212336459a9c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Aug 2016 13:40:29 +0200
Subject: [PATCH 155/623] fixed sporadically test cases assertion error
 "AssertionError: 3 != 4" - because multi-threaded, occurred if
 getFailTicket() called after 3th failure (before 4th failure was monitored),
 so we will wait in new method waitFailTotal (instead of isFilled) for
 expected failure count...

---
 fail2ban/tests/filtertestcase.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 7907965b..fe64e55e 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -912,6 +912,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log")
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
+			self._failTotal = 0
 			# UUID used to ensure that only meeages generated
 			# as part of this test are picked up by the filter
 			self.test_uuid = str(uuid.uuid4())
@@ -939,6 +940,15 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			return "MonitorJournalFailures%s(%s)" \
 			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
 
+		def waitFailTotal(self, count, delay=1.):
+			"""Wait up to `delay` sec to assure that expected failure `count` reached
+			"""
+			ret = Utils.wait_for(
+				lambda: self.filter.failManager.getFailTotal() >= self._failTotal + count and self.jail.isFilled(),
+				_maxWaitTime(delay))
+			self._failTotal += count
+			return ret
+
 		def isFilled(self, delay=1.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
@@ -949,8 +959,9 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
 
 		def assert_correct_ban(self, test_ip, test_attempts):
-			self.assertTrue(self.isFilled(_maxWaitTime(10))) # give Filter a chance to react
+			self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react
 			ticket = self.jail.getFailTicket()
+			self.assertTrue(ticket)
 
 			attempts = ticket.getAttempt()
 			ip = ticket.getIP()

From 67e4ac5c666e060d59eb22bd37edcad31571e1a8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Aug 2016 13:45:50 +0200
Subject: [PATCH 156/623] common monitor test case class, that unite some
 duplicate code

---
 fail2ban/tests/filtertestcase.py | 60 ++++++++++++++++----------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index fe64e55e..b6faf836 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -695,6 +695,32 @@ class LogFileMonitor(LogCaptureTestCase):
 		self.assertEqual(self.filter.failManager.getFailTotal(), 3)
 
 
+class CommonMonitorTestCase(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		self._failTotal = 0
+
+	def waitFailTotal(self, count, delay=1.):
+		"""Wait up to `delay` sec to assure that expected failure `count` reached
+		"""
+		ret = Utils.wait_for(
+			lambda: self.filter.failManager.getFailTotal() >= self._failTotal + count and self.jail.isFilled(),
+			_maxWaitTime(delay))
+		self._failTotal += count
+		return ret
+
+	def isFilled(self, delay=1.):
+		"""Wait up to `delay` sec to assure that it was modified or not
+		"""
+		return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
+
+	def isEmpty(self, delay=_maxWaitTime(5)):
+		"""Wait up to `delay` sec to assure that it empty again
+		"""
+		return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
+
+
 def get_monitor_failures_testcase(Filter_):
 	"""Generator of TestCase's for different filters/backends
 	"""
@@ -703,11 +729,12 @@ def get_monitor_failures_testcase(Filter_):
 	testclass_name = tempfile.mktemp(
 		'fail2ban', 'monitorfailures_%s' % (Filter_.__name__,))
 
-	class MonitorFailures(unittest.TestCase):
+	class MonitorFailures(CommonMonitorTestCase):
 		count = 0
 
 		def setUp(self):
 			"""Call before every test case."""
+			super(MonitorFailures, self).setUp()
 			setUpMyTime()
 			self.filter = self.name = 'NA'
 			self.name = '%s-%d' % (testclass_name, self.count)
@@ -737,11 +764,6 @@ def get_monitor_failures_testcase(Filter_):
 			#time.sleep(0.2)			  # Give FS time to ack the removal
 			pass
 
-		def isFilled(self, delay=1.):
-			"""Wait up to `delay` sec to assure that it was modified or not
-			"""
-			return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
-
 		def _sleep_4_poll(self):
 			# Since FilterPoll relies on time stamps and some
 			# actions might be happening too fast in the tests,
@@ -749,10 +771,6 @@ def get_monitor_failures_testcase(Filter_):
 			if isinstance(self.filter, FilterPoll):
 				Utils.wait_for(self.filter.isAlive, _maxWaitTime(5))
 
-		def isEmpty(self, delay=_maxWaitTime(5)):
-			# shorter wait time for not modified status
-			return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
-
 		def assert_correct_last_attempt(self, failures, count=None):
 			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
 			_assert_correct_last_attempt(self, self.jail, failures, count=count)
@@ -906,13 +924,13 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 	"""Generator of TestCase's for journal based filters/backends
 	"""
 
-	class MonitorJournalFailures(unittest.TestCase):
+	class MonitorJournalFailures(CommonMonitorTestCase):
 		def setUp(self):
 			"""Call before every test case."""
+			super(MonitorJournalFailures, self).setUp()
 			self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log")
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
-			self._failTotal = 0
 			# UUID used to ensure that only meeages generated
 			# as part of this test are picked up by the filter
 			self.test_uuid = str(uuid.uuid4())
@@ -940,24 +958,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			return "MonitorJournalFailures%s(%s)" \
 			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
 
-		def waitFailTotal(self, count, delay=1.):
-			"""Wait up to `delay` sec to assure that expected failure `count` reached
-			"""
-			ret = Utils.wait_for(
-				lambda: self.filter.failManager.getFailTotal() >= self._failTotal + count and self.jail.isFilled(),
-				_maxWaitTime(delay))
-			self._failTotal += count
-			return ret
-
-		def isFilled(self, delay=1.):
-			"""Wait up to `delay` sec to assure that it was modified or not
-			"""
-			return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
-
-		def isEmpty(self, delay=_maxWaitTime(5)):
-			# shorter wait time for not modified status
-			return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
-
 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react
 			ticket = self.jail.getFailTicket()

From f91ec84bfbbc4a62cefa912ffd92dfaf22e925f0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Aug 2016 15:46:44 +0200
Subject: [PATCH 157/623] prevent theoretical time-related assertion errors for
 other filters, replace isFilled with waitFailTotal (known expected failure
 count)

---
 fail2ban/tests/filtertestcase.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index b6faf836..4a20cb07 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -83,7 +83,7 @@ def _killfile(f, name):
 
 
 def _maxWaitTime(wtime):
-	if unittest.F2B.fast:
+	if unittest.F2B.fast: # pragma: no cover
 		wtime /= 10
 	return wtime
 
@@ -715,7 +715,7 @@ class CommonMonitorTestCase(unittest.TestCase):
 		"""
 		return Utils.wait_for(self.jail.isFilled, _maxWaitTime(delay))
 
-	def isEmpty(self, delay=_maxWaitTime(5)):
+	def isEmpty(self, delay=5):
 		"""Wait up to `delay` sec to assure that it empty again
 		"""
 		return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
@@ -772,7 +772,7 @@ def get_monitor_failures_testcase(Filter_):
 				Utils.wait_for(self.filter.isAlive, _maxWaitTime(5))
 
 		def assert_correct_last_attempt(self, failures, count=None):
-			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
+			self.assertTrue(self.waitFailTotal(count if count else failures[1], 10))
 			_assert_correct_last_attempt(self, self.jail, failures, count=count)
 
 		def test_grow_file(self):
@@ -788,7 +788,7 @@ def get_monitor_failures_testcase(Filter_):
 
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5)
 			self.assertTrue(self.isFilled(10))
-			# so we sleep for up to 2 sec for it not to become empty,
+			# so we sleep a bit for it not to become empty,
 			# and meanwhile pass to other thread(s) and filter should
 			# have gathered new failures and passed them into the
 			# DummyJail

From cd6b5285427856e3cb18e3ef870ed047a8e21dec Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Aug 2016 16:37:49 +0200
Subject: [PATCH 158/623] new test-cases option "--verbosity" to set numeric
 level of verbosity during test cases; travis uses same verbosity (2) running
 in python3 as from setup (python2)

---
 .travis.yml             |  4 ++--
 bin/fail2ban-testcases  |  4 ++++
 fail2ban/tests/utils.py | 22 ++++++++++++----------
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8aa781be..3f07cc51 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -38,8 +38,8 @@ before_script:
 script:
   # Keep the legacy setup.py test approach of checking coverage for python2
   - if [[ "$F2B_PY_2" ]]; then coverage run setup.py test; fi
-  # Coverage doesn't pick up setup.py test with python3, so run it directly
-  - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi
+  # Coverage doesn't pick up setup.py test with python3, so run it directly (with same verbosity as from setup)
+  - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi
   # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
   - sudo $VENV_BIN/pip install .
   # Doc files should get installed on Travis under Linux
diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index 4a93ac95..cfd95055 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -61,6 +61,10 @@ def get_opt_parser():
 			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
+		Option('-v', "--verbosity", action="store",
+			   dest="verbosity", type=int,
+			   default=None,
+			   help="Set numerical level of verbosity (0..4)"),
 		Option("--log-direct", action="store_false",
 			   dest="log_lazy",
 			   default=True,
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 232da780..764536bf 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -64,7 +64,7 @@ os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname(
 class DefaultTestOptions(optparse.Values):
 	def __init__(self):
 		self.__dict__ = {
-			'log_level': None, 'log_lazy': True, 
+			'log_level': None, 'verbosity': None, 'log_lazy': True, 
 			'log_traceback': None, 'full_traceback': None,
 			'fast': False, 'memory_db': False, 'no_gamin': False,
 			'no_network': False, 'negate_re': False
@@ -78,15 +78,17 @@ def initProcess(opts):
 	logSys = getLogger("fail2ban")
 
 	# Numerical level of verbosity corresponding to a log "level"
-	verbosity = {'heavydebug': 4,
-				 'debug': 3,
-				 'info': 2,
-				 'notice': 2,
-				 'warning': 1,
-				 'error': 1,
-				 'critical': 0,
-				 None: 1}[opts.log_level]
-	opts.verbosity = verbosity
+	verbosity = opts.verbosity
+	if verbosity is None:
+		verbosity = {'heavydebug': 4,
+					 'debug': 3,
+					 'info': 2,
+					 'notice': 2,
+					 'warning': 1,
+					 'error': 1,
+					 'critical': 0,
+					 None: 1}[opts.log_level]
+		opts.verbosity = verbosity
 
 	if opts.log_level is not None: # pragma: no cover
 		# so we had explicit settings

From 9aca34d3b19a683fe0415b995bd1e77c6b08ae87 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Aug 2016 17:48:57 +0200
Subject: [PATCH 159/623] normalize name of the test cases: method and class
 name fixed for MonitorJournalFailures

---
 fail2ban/tests/filtertestcase.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 4a20cb07..15058d76 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -727,7 +727,7 @@ def get_monitor_failures_testcase(Filter_):
 
 	# add Filter_'s name so we could easily identify bad cows
 	testclass_name = tempfile.mktemp(
-		'fail2ban', 'monitorfailures_%s' % (Filter_.__name__,))
+		'fail2ban', 'monitorfailures_%s_' % (Filter_.__name__,))
 
 	class MonitorFailures(CommonMonitorTestCase):
 		count = 0
@@ -923,6 +923,8 @@ def get_monitor_failures_testcase(Filter_):
 def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 	"""Generator of TestCase's for journal based filters/backends
 	"""
+	
+	testclass_name = "monitorjournalfailures_%s" % (Filter_.__name__,)
 
 	class MonitorJournalFailures(CommonMonitorTestCase):
 		def setUp(self):
@@ -934,7 +936,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			# UUID used to ensure that only meeages generated
 			# as part of this test are picked up by the filter
 			self.test_uuid = str(uuid.uuid4())
-			self.name = "monitorjournalfailures-%s" % self.test_uuid
+			self.name = "%s-%s" % (testclass_name, self.test_uuid)
 			self.filter.addJournalMatch([
 				"SYSLOG_IDENTIFIER=fail2ban-testcases",
 				"TEST_FIELD=1",
@@ -954,10 +956,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.filter.join()		  # wait for the thread to terminate
 			pass
 
-		def __str__(self):
-			return "MonitorJournalFailures%s(%s)" \
-			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
-
 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react
 			ticket = self.jail.getFailTicket()
@@ -1029,6 +1027,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			# we should detect the failures
 			self.assertTrue(self.isFilled(10))
 
+	MonitorJournalFailures.__name__ = "MonitorJournalFailures<%s>(%s)" \
+			  % (Filter_.__name__, testclass_name)
 	return MonitorJournalFailures
 
 

From f7f618b15d92969014d73857ebbf587653c6cf18 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 18 Aug 2016 14:22:38 +0200
Subject: [PATCH 160/623] Let know the version (and python), also in travis (no
 output if verbosity is 0)

---
 bin/fail2ban-testcases | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index cfd95055..d8c40727 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -105,7 +105,7 @@ verbosity = opts.verbosity
 #
 # Let know the version
 #
-if opts.log_level != logging.CRITICAL: # pragma: no cover
+if opts.verbosity != 0: # pragma: no cover
 	print("Fail2ban %s test suite. Python %s. Please wait..." \
 			% (version, str(sys.version).replace('\n', '')))
 

From af126eb308712bfb95829d618800be52ea73a57d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 18 Aug 2016 14:40:53 +0200
Subject: [PATCH 161/623] simplify fail2ban-testcases: move some code pieces
 inclusive option parser from fail2ban-testcases to tests/utils.py (+
 coverage)

---
 bin/fail2ban-testcases  | 58 +++--------------------------------------
 fail2ban/tests/utils.py | 51 +++++++++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 55 deletions(-)

diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases
index d8c40727..5539d4c6 100755
--- a/bin/fail2ban-testcases
+++ b/bin/fail2ban-testcases
@@ -37,11 +37,9 @@ if os.path.exists("fail2ban/__init__.py"):
 	sys.path.insert(0, ".")
 from fail2ban.version import version
 
-from fail2ban.tests.utils import initProcess, gatherTests
+from fail2ban.tests.utils import getOptParser, initProcess, gatherTests
 from fail2ban.setup import updatePyExec
 
-from optparse import OptionParser, Option
-
 # Update fail2ban-python env to current python version (where f2b-modules located/installed)
 bindir = os.path.dirname(
 	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
@@ -49,52 +47,7 @@ bindir = os.path.dirname(
 )
 updatePyExec(bindir)
 
-def get_opt_parser():
-	# use module docstring for help output
-	p = OptionParser(
-				usage="%s [OPTIONS] [regexps]\n" % sys.argv[0] + __doc__,
-				version="%prog " + version)
-
-	p.add_options([
-		Option('-l', "--log-level", type="choice",
-			   dest="log_level",
-			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
-			   default=None,
-			   help="Log level for the logger to use during running tests"),
-		Option('-v', "--verbosity", action="store",
-			   dest="verbosity", type=int,
-			   default=None,
-			   help="Set numerical level of verbosity (0..4)"),
-		Option("--log-direct", action="store_false",
-			   dest="log_lazy",
-			   default=True,
-			   help="Prevent lazy logging inside tests"),
-		Option('-n', "--no-network", action="store_true",
-			   dest="no_network",
-			   help="Do not run tests that require the network"),
-		Option('-g', "--no-gamin", action="store_true",
-			   dest="no_gamin",
-			   help="Do not run tests that require the gamin"),
-		Option('-m', "--memory-db", action="store_true",
-			   dest="memory_db",
-			   help="Run database tests using memory instead of file"),
-		Option('-f', "--fast", action="store_true",
-			   dest="fast",
-			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
-		Option('-i', "--ignore", action="store_true",
-			   dest="negate_re",
-			   help="negate [regexps] filter to ignore tests matched specified regexps"),
-		Option("-t", "--log-traceback", action='store_true',
-			   help="Enrich log-messages with compressed tracebacks"),
-		Option("--full-traceback", action='store_true',
-			   help="Either to make the tracebacks full, not compressed (as by default)"),
-
-		])
-
-	return p
-
-parser = get_opt_parser()
-(opts, regexps) = parser.parse_args()
+(opts, regexps) = getOptParser(__doc__).parse_args()
 
 #
 # Process initialization corresponding options (logging, default options, etc.)
@@ -103,13 +56,10 @@ opts = initProcess(opts)
 verbosity = opts.verbosity
 
 #
-# Let know the version
+# Gather tests (and filter corresponding options)
 #
-if opts.verbosity != 0: # pragma: no cover
-	print("Fail2ban %s test suite. Python %s. Please wait..." \
-			% (version, str(sys.version).replace('\n', '')))
-
 tests = gatherTests(regexps, opts)
+
 #
 # Run the tests
 #
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 764536bf..6d0828d4 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -43,6 +43,7 @@ from ..server.mytime import MyTime
 from ..server.utils import Utils
 # for action_d.test_smtp :
 from ..server import asyncserver
+from ..version import version
 
 
 logSys = getLogger(__name__)
@@ -73,6 +74,49 @@ class DefaultTestOptions(optparse.Values):
 #
 # Initialization
 #
+def getOptParser(doc=""):
+	Option = optparse.Option
+	# use module docstring for help output
+	p = optparse.OptionParser(
+				usage="%s [OPTIONS] [regexps]\n" % sys.argv[0] + doc,
+				version="%prog " + version)
+
+	p.add_options([
+		Option('-l', "--log-level", type="choice",
+			   dest="log_level",
+			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
+			   default=None,
+			   help="Log level for the logger to use during running tests"),
+		Option('-v', "--verbosity", action="store",
+			   dest="verbosity", type=int,
+			   default=None,
+			   help="Set numerical level of verbosity (0..4)"),
+		Option("--log-direct", action="store_false",
+			   dest="log_lazy",
+			   default=True,
+			   help="Prevent lazy logging inside tests"),
+		Option('-n', "--no-network", action="store_true",
+			   dest="no_network",
+			   help="Do not run tests that require the network"),
+		Option('-g', "--no-gamin", action="store_true",
+			   dest="no_gamin",
+			   help="Do not run tests that require the gamin"),
+		Option('-m', "--memory-db", action="store_true",
+			   dest="memory_db",
+			   help="Run database tests using memory instead of file"),
+		Option('-f', "--fast", action="store_true",
+			   dest="fast",
+			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
+		Option('-i', "--ignore", action="store_true",
+			   dest="negate_re",
+			   help="negate [regexps] filter to ignore tests matched specified regexps"),
+		Option("-t", "--log-traceback", action='store_true',
+			   help="Enrich log-messages with compressed tracebacks"),
+		Option("--full-traceback", action='store_true',
+			   help="Either to make the tracebacks full, not compressed (as by default)"),
+		])
+	return p
+
 def initProcess(opts):
 	# Logger:
 	logSys = getLogger("fail2ban")
@@ -123,7 +167,12 @@ def initProcess(opts):
 	#
 	stdout.setFormatter(Formatter(fmt))
 	logSys.addHandler(stdout)
-	#
+
+	# Let know the version
+	if opts.verbosity != 0:
+		print("Fail2ban %s test suite. Python %s. Please wait..." \
+				% (version, str(sys.version).replace('\n', '')))
+
 	return opts;
 
 

From 23c14acec070dcf80aa02c8fd58463eae73d207b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 1 Sep 2016 16:13:15 +0200
Subject: [PATCH 162/623] update `_0.10/systemd-journal-path-gh-1408` (after
 merge `_0.9/systemd-journal-path-gh-1408` to 0.10)

---
 fail2ban/server/filtersystemd.py |  6 +++--
 fail2ban/tests/filtertestcase.py | 42 ++++++++++++++++++++++++++------
 2 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 9c4c7865..2ab2f6b4 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -274,8 +274,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			if self.idle:
 				# because journal.wait will returns immediatelly if we have records in journal,
 				# just wait a little bit here for not idle, to prevent hi-load:
-				time.sleep(self.sleeptime)
-				continue
+				if not Utils.wait_for(lambda: not self.idle, 
+					self.sleeptime * 10, self.sleeptime
+				):
+					continue
 			self.__modified = 0
 			while self.active:
 				logentry = None
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 0f338fbe..40d37bb6 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -84,7 +84,7 @@ def _killfile(f, name):
 
 def _maxWaitTime(wtime):
 	if unittest.F2B.fast: # pragma: no cover
-		wtime /= 10
+		wtime /= 10.0
 	return wtime
 
 
@@ -958,12 +958,40 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				self.filter.join()		  # wait for the thread to terminate
 				pass
 
-		def testJournalFlagsArg(self):
-			self._initFilter(journalflags=2) # journal.RUNTIME_ONLY
+		def waitForTicks(self, ticks, delay=2.):
+			"""Wait up to `delay` sec to assure that it was modified or not
+			"""
+			last_ticks = self.filter.ticks
+			return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay))
 
-		def __str__(self):
-			return "MonitorJournalFailures%s(%s)" \
-			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
+		def _getRuntimeJournal(self):
+			# retrieve current system journal path
+			tmp = Utils.executeCmd('find "$(systemd-path system-runtime-logs)" -name system.journal', 
+				timeout=10, shell=True, output=True);
+			self.assertTrue(tmp)
+			return str(tmp[1].decode('utf-8')).split('\n')[0]
+
+		def testJournalFilesArg(self):
+			# retrieve current system journal path
+			jrnlfile = self._getRuntimeJournal()
+			self._initFilter(journalfiles=jrnlfile)
+
+		def testJournalPathArg(self):
+			# retrieve current system journal path
+			jrnlpath = self._getRuntimeJournal()
+			jrnlpath = os.path.dirname(jrnlpath)
+			self._initFilter(journalpath=jrnlpath)
+			self.filter.seekToTime(
+				datetime.datetime.now() - datetime.timedelta(days=1)
+			)
+			self.filter.start()
+			self.waitForTicks(2)
+			self.assertTrue(self.isEmpty(1))
+			self.assertEqual(len(self.jail), 0)
+			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+
+		def testJournalFlagsArg(self):
+			self._initFilter(journalflags=0) # e. g. 2 - journal.RUNTIME_ONLY
 
 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react
@@ -1027,7 +1055,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, n=5, skip=5)
 			# so we should get no more failures detected
-			self.assertTrue(self.isEmpty(_maxWaitTime(10)))
+			self.assertTrue(self.isEmpty(10))
 
 			# but then if we add it back again
 			self.filter.addJournalMatch([

From d974ebd5814b8703a6776a1df1b6fcb2bec048b9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 1 Sep 2016 19:38:40 +0200
Subject: [PATCH 163/623] rewritten idle handling for all filters, test cases
 extended to work in idle mode also (should not find/process failures)

---
 fail2ban/server/filter.py          |  2 ++
 fail2ban/server/filtergamin.py     | 12 +++++--
 fail2ban/server/filterpoll.py      |  7 +++--
 fail2ban/server/filterpyinotify.py | 50 ++++++++++++++++--------------
 fail2ban/server/filtersystemd.py   |  3 +-
 fail2ban/tests/filtertestcase.py   | 40 ++++++++++++++++++++----
 fail2ban/tests/utils.py            |  8 ++---
 7 files changed, 83 insertions(+), 39 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 318d8e49..4f30719d 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -85,6 +85,8 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## External command
 		self.__ignoreCommand = False
+		## Ticks counter
+		self.ticks = 0
 
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index 6b562a5c..39793067 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -69,6 +69,7 @@ class FilterGamin(FileFilter):
 			logSys.debug("File changed: " + path)
 			self.__modified = True
 
+		self.ticks += 1
 		self._process_file(path)
 
 	def _process_file(self, path):
@@ -105,7 +106,7 @@ class FilterGamin(FileFilter):
 	def _handleEvents(self):
 		ret = False
 		mon = self.monitor
-		while mon and mon.event_pending():
+		while mon and mon.event_pending() > 0:
 			mon.handle_events()
 			mon = self.monitor
 			ret = True
@@ -122,9 +123,14 @@ class FilterGamin(FileFilter):
 		# Gamin needs a loop to collect and dispatch events
 		while self.active:
 			if self.idle:
-				time.sleep(self.sleeptime)
-				continue
+				# wait a little bit here for not idle, to prevent hi-load:
+				if not Utils.wait_for(lambda: not self.idle, 
+					self.sleeptime * 10, self.sleeptime
+				):
+					self.ticks += 1
+					continue
 			Utils.wait_for(self._handleEvents, self.sleeptime)
+			self.ticks += 1
 		logSys.debug(self.jail.name + ": filter terminated")
 		return True
 
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index c7b04970..bebdcad2 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -30,8 +30,9 @@ import time
 from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger
-from ..server.utils import Utils
+
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -101,8 +102,9 @@ class FilterPoll(FileFilter):
 						   self.idle, self.getLogCount())
 			if self.idle:
 				if not Utils.wait_for(lambda: not self.idle, 
-					self.sleeptime * 100, self.sleeptime
+					self.sleeptime * 10, self.sleeptime
 				):
+					self.ticks += 1
 					continue
 			# Get file modification
 			modlst = []
@@ -111,6 +113,7 @@ class FilterPoll(FileFilter):
 				self.getFailures(filename)
 				self.__modified = True
 
+			self.ticks += 1
 			if self.__modified:
 				try:
 					while True:
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 7bd638d9..2116cb92 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -32,6 +32,7 @@ import pyinotify
 from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger
 
 
@@ -92,7 +93,9 @@ class FilterPyinotify(FileFilter):
 				self._delFileWatcher(path)
 				# place a new one
 				self._addFileWatcher(path)
-
+		# do nothing if idle:
+		if self.idle:
+			return
 		self._process_file(path)
 
 	def _process_file(self, path):
@@ -159,6 +162,25 @@ class FilterPyinotify(FileFilter):
 			self.__monitor.rm_watch(wdInt)
 			logSys.debug("Removed monitor for the parent directory %s", path_dir)
 
+	# pyinotify.ProcessEvent default handler:
+	def __process_default(self, event):
+		try:
+			self.callback(event, origin='Default ')
+		except Exception as e:
+			logSys.error("Error in FilterPyinotify callback: %s",
+				e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
+		self.ticks += 1
+
+	# slow check events while idle:
+	def __check_events(self, *args, **kwargs):
+		if self.idle:
+			if Utils.wait_for(lambda: not self.idle, 
+				self.sleeptime * 10, self.sleeptime
+			):
+				pass
+		self.ticks += 1
+		return pyinotify.ThreadedNotifier.check_events(self.__notifier, *args, **kwargs)
+
 	##
 	# Main loop.
 	#
@@ -166,12 +188,13 @@ class FilterPyinotify(FileFilter):
 	# loop is necessary
 
 	def run(self):
+		prcevent = pyinotify.ProcessEvent()
+		prcevent.process_default = self.__process_default
 		self.__notifier = pyinotify.ThreadedNotifier(self.__monitor,
-			ProcessPyinotify(self))
+			prcevent, timeout=self.sleeptime)
+		self.__notifier.check_events = self.__check_events
 		self.__notifier.start()
 		logSys.debug("pyinotifier started for %s.", self.jail.name)
-		# TODO: verify that there is nothing really to be done for
-		#       idle jails
 		return True
 
 	##
@@ -191,22 +214,3 @@ class FilterPyinotify(FileFilter):
 	def __cleanup(self):
 		self.__notifier = None
 		self.__monitor = None
-
-
-class ProcessPyinotify(pyinotify.ProcessEvent):
-	def __init__(self, FileFilter, **kargs):
-		#super(ProcessPyinotify, self).__init__(**kargs)
-		# for some reason root class _ProcessEvent is old-style (is
-		# not derived from object), so to play safe let's avoid super
-		# for now, and call superclass directly
-		pyinotify.ProcessEvent.__init__(self, **kargs)
-		self.__FileFilter = FileFilter
-		pass
-
-	# just need default, since using mask on watch to limit events
-	def process_default(self, event):
-		try:
-			self.__FileFilter.callback(event, origin='Default ')
-		except Exception as e:
-			logSys.error("Error in FilterPyinotify callback: %s",
-				e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 2ab2f6b4..3d41dce9 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -33,6 +33,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204':
 from .failmanager import FailManagerEmpty
 from .filter import JournalFilter
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger, logging, splitwords
 
 # Gets the instance of the logger.
@@ -61,7 +62,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		self.__journal = journal.Reader(**jrnlargs)
 		self.__matches = []
 		self.setDatePattern(None)
-		self.ticks = 0
 		logSys.debug("Created FilterSystemd")
 
 	@staticmethod
@@ -277,6 +277,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				if not Utils.wait_for(lambda: not self.idle, 
 					self.sleeptime * 10, self.sleeptime
 				):
+					self.ticks += 1
 					continue
 			self.__modified = 0
 			while self.active:
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 40d37bb6..c4801282 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -720,6 +720,12 @@ class CommonMonitorTestCase(unittest.TestCase):
 		"""
 		return Utils.wait_for(self.jail.isEmpty, _maxWaitTime(delay))
 
+	def waitForTicks(self, ticks, delay=2.):
+		"""Wait up to `delay` sec to assure that it was modified or not
+		"""
+		last_ticks = self.filter.ticks
+		return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay))
+
 
 def get_monitor_failures_testcase(Filter_):
 	"""Generator of TestCase's for different filters/backends
@@ -776,6 +782,16 @@ def get_monitor_failures_testcase(Filter_):
 			_assert_correct_last_attempt(self, self.jail, failures, count=count)
 
 		def test_grow_file(self):
+			self._test_grow_file()
+
+		def test_grow_file_in_idle(self):
+			self._test_grow_file(True)
+
+		def _test_grow_file(self, idle=False):
+			if idle:
+				self.filter.sleeptime /= 100.0
+				self.filter.idle = True
+				self.waitForTicks(1)
 			# suck in lines from this sample log file
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 
@@ -787,6 +803,10 @@ def get_monitor_failures_testcase(Filter_):
 			# since it should have not been enough
 
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5)
+			if idle:
+				self.waitForTicks(1)
+				self.assertTrue(self.isEmpty(1))
+				return
 			self.assertTrue(self.isFilled(10))
 			# so we sleep a bit for it not to become empty,
 			# and meanwhile pass to other thread(s) and filter should
@@ -958,12 +978,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				self.filter.join()		  # wait for the thread to terminate
 				pass
 
-		def waitForTicks(self, ticks, delay=2.):
-			"""Wait up to `delay` sec to assure that it was modified or not
-			"""
-			last_ticks = self.filter.ticks
-			return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay))
-
 		def _getRuntimeJournal(self):
 			# retrieve current system journal path
 			tmp = Utils.executeCmd('find "$(systemd-path system-runtime-logs)" -name system.journal', 
@@ -1006,8 +1020,18 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.assertEqual(attempts, test_attempts)
 
 		def test_grow_file(self):
+			self._test_grow_file()
+
+		def test_grow_file_in_idle(self):
+			self._test_grow_file(True)
+
+		def _test_grow_file(self, idle=False):
 			self._initFilter()
 			self.filter.start()
+			if idle:
+				self.filter.sleeptime /= 100.0
+				self.filter.idle = True
+				self.waitForTicks(1)
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 
 			# Now let's feed it with entries from the file
@@ -1020,6 +1044,10 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, skip=2, n=3)
+			if idle:
+				self.waitForTicks(1)
+				self.assertTrue(self.isEmpty(1))
+				return
 			self.assertTrue(self.isFilled(10))
 			# so we sleep for up to 6 sec for it not to become empty,
 			# and meanwhile pass to other thread(s) and filter should
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 6d0828d4..4cbf4354 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -430,16 +430,16 @@ def gatherTests(regexps=None, opts=None):
 		# because gamin can be very slow on some platforms (and can produce many failures 
 		# with fast sleep interval) - skip it by fast run:
 		if unittest.F2B.fast or unittest.F2B.no_gamin: # pragma: no cover
-			raise Exception('Skip, fast: %s, no_gamin: %s' % (unittest.F2B.fast, unittest.F2B.no_gamin))
+			raise ImportError('Skip, fast: %s, no_gamin: %s' % (unittest.F2B.fast, unittest.F2B.no_gamin))
 		from ..server.filtergamin import FilterGamin
 		filters.append(FilterGamin)
-	except Exception, e: # pragma: no cover
+	except ImportError, e: # pragma: no cover
 		logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e)
 
 	try:
 		from ..server.filterpyinotify import FilterPyinotify
 		filters.append(FilterPyinotify)
-	except Exception, e: # pragma: no cover
+	except ImportError, e: # pragma: no cover
 		logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e)
 
 	for Filter_ in filters:
@@ -448,7 +448,7 @@ def gatherTests(regexps=None, opts=None):
 	try: # pragma: systemd no cover
 		from ..server.filtersystemd import FilterSystemd
 		tests.addTest(unittest.makeSuite(filtertestcase.get_monitor_failures_journal_testcase(FilterSystemd)))
-	except Exception, e: # pragma: no cover
+	except ImportError, e: # pragma: no cover
 		logSys.warning("I: Skipping systemd backend testing. Got exception '%s'" % e)
 
 	# Server test for logging elements which break logging used to support

From 90f9009959320d2663783b40ec496cb19862df1f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 5 Sep 2016 16:08:26 +0200
Subject: [PATCH 164/623] fixes wrong date (invalid format or value)
 recognition if template available (+ test cases extended)

---
 fail2ban/server/datedetector.py               | 17 ++++++++++-------
 fail2ban/tests/files/logs/zzz-generic-example | 16 ++++++++++++++++
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 0c5da94b..5281fc59 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -224,12 +224,15 @@ class DateDetector(object):
 		if timeMatch:
 			template = timeMatch[1]
 			if template is not None:
-				date = template.getDate(line, timeMatch[0])
-				if date is not None:
-					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "Got time %f for \"%r\" using template %s",
-							date[0], date[1].group(), template.name)
-					return date
+				try:
+					date = template.getDate(line, timeMatch[0])
+					if date is not None:
+						if logSys.getEffectiveLevel() <= logLevel:
+							logSys.log(logLevel, "Got time %f for %r using template %s",
+								date[0], date[1].group(), template.name)
+						return date
+				except ValueError:
+					return None
 		with self.__lock:
 			for template in self.__templates:
 				try:
@@ -237,7 +240,7 @@ class DateDetector(object):
 					if date is None:
 						continue
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "Got time %f for \"%r\" using template %s", 
+						logSys.log(logLevel, "Got time %f for %r using template %s", 
 							date[0], date[1].group(), template.name)
 					return date
 				except ValueError: # pragma: no cover
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index e3480b63..2a4fb789 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -29,3 +29,19 @@ Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-
 # -- the same as above with brackets as date ambit --
 # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
 [Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
+
+# -- wrong time direct in journal-line (used last known date):
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" }
+0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.1
+# -- wrong time after newline in message (plist without escaped newlines):
+# failJSON: { "match": false }
+Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
+# failJSON: { "match": false }
+             absentCircleWithNoReason = 0;
+# failJSON: { "match": false }
+             applicationDate = "0000-12-30 00:00:00 +0000";
+# failJSON: { "match": false }
+}
+# -- wrong time direct in journal-line (used last known date):
+# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
+0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2

From 6454d0cd75005ef5ff52b14aa5b17c3942d9c305 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 5 Sep 2016 16:09:21 +0200
Subject: [PATCH 165/623] prevents to stop processing after first unhandled
 exception inside processLineAndAdd (filter going idle after 100th error);

---
 fail2ban/server/filter.py        | 55 ++++++++++++++++++++------------
 fail2ban/tests/filtertestcase.py | 21 ++++++++++++
 2 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 318d8e49..6be71355 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -85,6 +85,8 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## External command
 		self.__ignoreCommand = False
+		## Error counter
+		self.__errors = 0
 
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
@@ -414,26 +416,39 @@ class Filter(JailThread):
 	def processLineAndAdd(self, line, date=None):
 		"""Processes the line for failures and populates failManager
 		"""
-		for element in self.processLine(line, date)[1]:
-			ip = element[1]
-			unixTime = element[2]
-			lines = element[3]
-			fail = {}
-			if len(element) > 4:
-				fail = element[4]
-			logSys.debug("Processing line with time:%s and ip:%s", 
-					unixTime, ip)
-			if unixTime < MyTime.time() - self.getFindTime():
-				logSys.debug("Ignore line since time %s < %s - %s", 
-					unixTime, MyTime.time(), self.getFindTime())
-				break
-			if self.inIgnoreIPList(ip, log_ignore=True):
-				continue
-			logSys.info(
-				"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
-			)
-			tick = FailTicket(ip, unixTime, lines, data=fail)
-			self.failManager.addFailure(tick)
+		try:
+			for element in self.processLine(line, date)[1]:
+				ip = element[1]
+				unixTime = element[2]
+				lines = element[3]
+				fail = {}
+				if len(element) > 4:
+					fail = element[4]
+				logSys.debug("Processing line with time:%s and ip:%s", 
+						unixTime, ip)
+				if unixTime < MyTime.time() - self.getFindTime():
+					logSys.debug("Ignore line since time %s < %s - %s", 
+						unixTime, MyTime.time(), self.getFindTime())
+					break
+				if self.inIgnoreIPList(ip, log_ignore=True):
+					continue
+				logSys.info(
+					"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
+				)
+				tick = FailTicket(ip, unixTime, lines, data=fail)
+				self.failManager.addFailure(tick)
+			# reset (halve) error counter (successfully processed line):
+			if self.__errors:
+				self.__errors //= 2
+		except Exception as e:
+			logSys.error("Failed to process line: %r, caught exception: %r", line, e,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			# incr error counter, stop processing this :
+			self.__errors += 1
+			if self.__errors >= 100:
+				logSys.error("Too many errors at once (%s), going idle", self.__errors)
+				self.__errors //= 2
+				self.idle = True
 
 	##
 	# Returns true if the line should be ignored.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 15058d76..ee9f9fb0 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -604,6 +604,27 @@ class LogFileMonitor(LogCaptureTestCase):
 		self.filter.getFailures(self.name)
 		self.assertLogged('Unable to open %s' % self.name)
 
+	def testErrorProcessLine(self):
+		## produce error with not callable processLine:
+		_org_processLine = self.filter.processLine
+		self.filter.processLine = None
+		for i in range(100):
+			self.file.write("line%d\n" % 1)
+		self.file.flush()
+		for i in range(100):
+			self.filter.getFailures(self.name)
+		self.assertLogged('Failed to process line:')
+		self.assertLogged('Too many errors at once')
+		self.pruneLog()
+		self.assertTrue(self.filter.idle)
+		self.filter.idle = False
+		self.filter.getFailures(self.name)
+		self.filter.processLine = _org_processLine
+		self.file.write("line%d\n" % 1)
+		self.file.flush()
+		self.filter.getFailures(self.name)
+		self.assertNotLogged('Failed to process line:')
+
 	def testRemovingFailRegex(self):
 		self.filter.delFailRegex(0)
 		self.assertNotLogged('Cannot remove regular expression. Index 0 is not valid')

From 1d6b546658b9bdbc74a190b7ea3ba6534aa40e44 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 5 Sep 2016 16:37:32 +0200
Subject: [PATCH 166/623] sleep a little bit in error case (to get around
 time-related errors, e.g. something like log-rotate, etc.)

---
 fail2ban/server/filter.py        | 5 ++++-
 fail2ban/tests/filtertestcase.py | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 6be71355..1f2950f3 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -29,6 +29,7 @@ import logging
 import os
 import re
 import sys
+import time
 
 from .failmanager import FailManagerEmpty, FailManager
 from .ipdns import DNSUtils, IPAddr
@@ -443,8 +444,10 @@ class Filter(JailThread):
 		except Exception as e:
 			logSys.error("Failed to process line: %r, caught exception: %r", line, e,
 				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-			# incr error counter, stop processing this :
+			# incr error counter, stop processing (going idle) after 100th error :
 			self.__errors += 1
+			# sleep a little bit (to get around time-related errors):
+			time.sleep(self.sleeptime)
 			if self.__errors >= 100:
 				logSys.error("Too many errors at once (%s), going idle", self.__errors)
 				self.__errors //= 2
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ee9f9fb0..2c9eddf3 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -605,6 +605,7 @@ class LogFileMonitor(LogCaptureTestCase):
 		self.assertLogged('Unable to open %s' % self.name)
 
 	def testErrorProcessLine(self):
+		self.filter.sleeptime /= 1000.0
 		## produce error with not callable processLine:
 		_org_processLine = self.filter.processLine
 		self.filter.processLine = None

From 87acd7a0fc17808c13ebe9fa7bfba0c2620311a3 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 6 Sep 2016 08:18:34 -0400
Subject: [PATCH 167/623] RF: Replace old fashioned "except E , e" with "except
 E as e" (Closes #1537)

0.10 specific
supplement to b875e51cd7b6cf4e257f88ce0a5c01b58a765e15 in master AKA 0.9
---
 fail2ban/client/configparserinc.py     |  2 +-
 fail2ban/client/configreader.py        |  2 +-
 fail2ban/client/fail2banclient.py      |  2 +-
 fail2ban/client/fail2bancmdline.py     |  2 +-
 fail2ban/client/fail2banregex.py       |  6 +++---
 fail2ban/client/fail2banserver.py      |  2 +-
 fail2ban/client/jailreader.py          |  2 +-
 fail2ban/server/action.py              |  2 +-
 fail2ban/server/database.py            | 10 +++++-----
 fail2ban/server/filter.py              | 14 +++++++-------
 fail2ban/server/filterpoll.py          |  2 +-
 fail2ban/server/filterpyinotify.py     |  2 +-
 fail2ban/server/ipdns.py               |  4 ++--
 fail2ban/server/jail.py                |  2 +-
 fail2ban/server/server.py              | 10 +++++-----
 fail2ban/server/transmitter.py         |  2 +-
 fail2ban/tests/clientreadertestcase.py |  4 ++--
 fail2ban/tests/samplestestcase.py      |  2 +-
 fail2ban/tests/servertestcase.py       |  2 +-
 fail2ban/tests/utils.py                |  6 +++---
 20 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 617ff8f6..d6cda7f7 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -171,7 +171,7 @@ after = 1.conf
 			parser, i = self._getSharedSCPWI(resource)
 			if not i:
 				return []
-		except UnicodeDecodeError, e:
+		except UnicodeDecodeError as e:
 			logSys.error("Error decoding config file '%s': %s" % (resource, e))
 			return []
 		
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index bcad59c3..643cdf3a 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -228,7 +228,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 				if not pOptions is None and optname in pOptions:
 					continue
 				values[optname] = v
-			except NoSectionError, e:
+			except NoSectionError as e:
 				if shouldExist:
 					raise
 				# No "Definition" section or wrong basedir
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 8be9e085..788906aa 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -381,7 +381,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 							elif not cmd == "":
 								try:
 									self.__processCommand(shlex.split(cmd))
-								except Exception, e: # pragma: no cover
+								except Exception as e: # pragma: no cover
 									if self._conf["verbose"] > 1:
 										logSys.exception(e)
 									else:
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 311c7af9..356255af 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -245,7 +245,7 @@ class Fail2banCmdLine():
 			ret = self.configurator.getOptions(jail)
 			self.configurator.convertToProtocol()
 			stream = self.configurator.getConfigStream()
-		except Exception, e:
+		except Exception as e:
 			logSys.error("Failed during configuration: %s" % e)
 			ret = False
 		return ret, stream
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index aa55a0ff..f36ca7b0 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -335,7 +335,7 @@ class Fail2banRegex(object):
 			if ret is not None:
 				found = True
 				regex = self._ignoreregex[ret].inc()
-		except RegexException, e:
+		except RegexException as e:
 			output( e )
 			return False
 		return found
@@ -352,7 +352,7 @@ class Fail2banRegex(object):
 				regex = self._failregex[match[0]]
 				regex.inc()
 				regex.appendIP(match)
-		except RegexException, e:
+		except RegexException as e:
 			output( e )
 			return False
 		except IndexError:
@@ -514,7 +514,7 @@ class Fail2banRegex(object):
 				output( "Use         log file : %s" % cmd_log )
 				output( "Use         encoding : %s" % self.encoding )
 				test_lines = self.file_lines_gen(hdlr)
-			except IOError, e:
+			except IOError as e:
 				output( e )
 				return False
 		elif cmd_log == "systemd-journal": # pragma: no cover
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 147e131f..85259e79 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -200,7 +200,7 @@ class Fail2banServer(Fail2banCmdLine):
 					exit(-1)
 				logSys.debug('Starting server done')
 
-		except Exception, e:
+		except Exception as e:
 			if self._conf["verbose"] > 1:
 				logSys.exception(e)
 			else:
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 13f611f7..e3c4731c 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -176,7 +176,7 @@ class JailReader(ConfigReader):
 							self.__actions.append(action)
 						else:
 							raise AttributeError("Unable to read action")
-				except Exception, e:
+				except Exception as e:
 					logSys.error("Error in action definition " + act)
 					logSys.debug("Caught exception: %s" % (e,))
 					return False
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index f579c31e..702ad397 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -284,7 +284,7 @@ class CommandAction(ActionBase):
 					res &= self.executeCmd(startCmd6, self.timeout)
 			if not res:
 				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
-		except ValueError, e:
+		except ValueError as e:
 			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
 
 	def ban(self, aInfo):
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 46d40051..4e2dddf1 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -42,7 +42,7 @@ if sys.version_info >= (3,):
 		try:
 			x = json.dumps(x, ensure_ascii=False).encode(
 				locale.getpreferredencoding(), 'replace')
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
 		return x
@@ -51,7 +51,7 @@ if sys.version_info >= (3,):
 		try:
 			x = json.loads(x.decode(
 				locale.getpreferredencoding(), 'replace'))
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
 		return x
@@ -70,7 +70,7 @@ else:
 		try:
 			x = json.dumps(_normalize(x), ensure_ascii=False).decode(
 				locale.getpreferredencoding(), 'replace')
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
 		return x
@@ -79,7 +79,7 @@ else:
 		try:
 			x = _normalize(json.loads(x.decode(
 				locale.getpreferredencoding(), 'replace')))
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
 		return x
@@ -176,7 +176,7 @@ class Fail2BanDb(object):
 
 			logSys.info(
 				"Connected to fail2ban persistent database '%s'", filename)
-		except sqlite3.OperationalError, e:
+		except sqlite3.OperationalError as e:
 			logSys.error(
 				"Error connecting to fail2ban persistent database '%s': %s",
 				filename, e.args[0])
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c357f6e6..70ae1a49 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -113,7 +113,7 @@ class Filter(JailThread):
 				logSys.warning(
 					"Mutliline regex set for jail '%s' "
 					"but maxlines not greater than 1")
-		except RegexException, e:
+		except RegexException as e:
 			logSys.error(e)
 			raise e
 
@@ -146,7 +146,7 @@ class Filter(JailThread):
 		try:
 			regex = Regex(value)
 			self.__ignoreRegex.append(regex)
-		except RegexException, e:
+		except RegexException as e:
 			logSys.error(e)
 			raise e 
 
@@ -583,7 +583,7 @@ class Filter(JailThread):
 										failRegex.getMatchedLines(), fail])
 								if not checkAllRegex:
 									break
-					except RegexException, e: # pragma: no cover - unsure if reachable
+					except RegexException as e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
 		return failList
 
@@ -729,15 +729,15 @@ class FileFilter(Filter):
 		try:
 			has_content = log.open()
 		# see http://python.org/dev/peps/pep-3151/
-		except IOError, e:
+		except IOError as e:
 			logSys.error("Unable to open %s" % filename)
 			logSys.exception(e)
 			return False
-		except OSError, e: # pragma: no cover - requires race condition to tigger this
+		except OSError as e: # pragma: no cover - requires race condition to tigger this
 			logSys.error("Error opening %s" % filename)
 			logSys.exception(e)
 			return False
-		except Exception, e: # pragma: no cover - Requires implemention error in FileContainer to generate
+		except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
 			logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
 			logSys.exception(e)
 			return False
@@ -750,7 +750,7 @@ class FileFilter(Filter):
 			# initial seek to start time using half-interval search algorithm:
 			try:
 				self.seekToTime(log, startTime)
-			except Exception, e: # pragma: no cover
+			except Exception as e: # pragma: no cover
 				logSys.error("Error during seek to start time in \"%s\"", filename)
 				raise
 				logSys.exception(e)
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index bebdcad2..336807d4 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -150,7 +150,7 @@ class FilterPoll(FileFilter):
 			logSys.debug("%s has been modified", filename)
 			self.__prevStats[filename] = stats
 			return True
-		except OSError, e:
+		except OSError as e:
 			logSys.error("Unable to get stat on %s because of: %s"
 						 % (filename, e))
 			self.__file404Cnt[filename] += 1
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 2116cb92..c5b5680b 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -45,7 +45,7 @@ if not hasattr(pyinotify, '__version__') \
 try:
 	manager = pyinotify.WatchManager()
 	del manager
-except Exception, e:
+except Exception as e:
 	raise ImportError("Pyinotify is probably not functional on this system: %s"
 					  % str(e))
 
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index cca63535..cba08d0c 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -70,7 +70,7 @@ class DNSUtils:
 				ip = IPAddr(result[4][0])
 				if ip.isValid:
 					ips.append(ip)
-		except socket.error, e:
+		except socket.error as e:
 			# todo: make configurable the expired time of cache entry:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
 			ips = list()
@@ -86,7 +86,7 @@ class DNSUtils:
 		# retrieve name
 		try:
 			v = socket.gethostbyaddr(ip)[0]
-		except socket.error, e:
+		except socket.error as e:
 			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
 			v = None
 		DNSUtils.CACHE_ipToName.set(ip, v)
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index 7659a7ad..92f8190e 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -109,7 +109,7 @@ class Jail(object):
 					logSys.info("Initiated %r backend" % b)
 				self.__actions = Actions(self)
 				return					# we are done
-			except ImportError, e: # pragma: no cover
+			except ImportError as e: # pragma: no cover
 				# Log debug if auto, but error if specific
 				logSys.log(
 					logging.DEBUG if backend == "auto" else logging.ERROR,
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 60eea1f3..db3918c7 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -138,7 +138,7 @@ class Server:
 			pidFile = open(pidfile, 'w')
 			pidFile.write("%s\n" % os.getpid())
 			pidFile.close()
-		except IOError, e:
+		except IOError as e:
 			logSys.error("Unable to create PID file: %s" % e)
 		
 		# Start the communication
@@ -146,13 +146,13 @@ class Server:
 		try:
 			self.__asyncServer = AsyncServer(self.__transm)
 			self.__asyncServer.start(sock, force)
-		except AsyncServerException, e:
+		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)
 		# Removes the PID file.
 		try:
 			logSys.debug("Remove PID file %s" % pidfile)
 			os.remove(pidfile)
-		except OSError, e:
+		except OSError as e:
 			logSys.error("Unable to remove PID file: %s" % e)
 		logSys.info("Exiting Fail2ban")
 	
@@ -604,7 +604,7 @@ class Server:
 			# the child gets a new PID, making it impossible for its PID to equal its
 			# PGID.
 			pid = os.fork()
-		except OSError, e:
+		except OSError as e:
 			return (False, (e.errno, e.strerror))	 # ERROR (return a tuple)
 		
 		if pid == 0:	   # The first child.
@@ -625,7 +625,7 @@ class Server:
 				# fork guarantees that the child is no longer a session leader, thus
 				# preventing the daemon from ever acquiring a controlling terminal.
 				pid = os.fork()		# Fork a second child.
-			except OSError, e:
+			except OSError as e:
 				return (False, (e.errno, e.strerror))  # ERROR (return a tuple)
 		
 			if (pid == 0):	  # The second child.
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index e4b9d81a..698dad34 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -56,7 +56,7 @@ class Transmitter:
 		try:
 			ret = self.__commandHandler(command)
 			ack = 0, ret
-		except Exception, e:
+		except Exception as e:
 			logSys.warning("Command %r has failed. Received %r"
 						% (command, e))
 			ack = 1, e
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 239989b7..e68523c2 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -421,7 +421,7 @@ class FilterReaderTest(unittest.TestCase):
 			# from testcase01
 			filterReader.get('Definition', 'failregex')
 			filterReader.get('Definition', 'ignoreregex')
-		except Exception, e: # pragma: no cover - failed if reachable
+		except Exception as e: # pragma: no cover - failed if reachable
 			self.fail('unexpected options after readexplicit: %s' % (e))
 
 
@@ -771,7 +771,7 @@ filter = testfilter1
 	def testLogPathSystemdBackend(self):
 		try: # pragma: systemd no cover
 			from ..server.filtersystemd import FilterSystemd
-		except Exception, e: # pragma: no cover
+		except Exception as e: # pragma: no cover
 			raise unittest.SkipTest("systemd python interface not available")
 		self._testLogPath(backend='systemd')
 		self._testLogPath(backend='systemd[journalflags=2]')
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 7af12f13..31b1812e 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -102,7 +102,7 @@ def testSampleRegexsFactory(name, basedir):
 			if jsonREMatch:
 				try:
 					faildata = json.loads(jsonREMatch.group(1))
-				except ValueError, e:
+				except ValueError as e:
 					raise ValueError("%s: %s:%i" %
 						(e, logFile.filename(), logFile.filelineno()))
 				line = next(logFile)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 2fd0efac..12bbb2f7 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1111,7 +1111,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
 					try:
 						cmdHandler(cmd)
-					except Exception, e:  # pragma: no cover
+					except Exception as e:  # pragma: no cover
 						self.fail("Command %r has failed. Received %r" % (cmd, e))
 
 			# jails = server._Server__jails
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 4cbf4354..bdc1c189 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -433,13 +433,13 @@ def gatherTests(regexps=None, opts=None):
 			raise ImportError('Skip, fast: %s, no_gamin: %s' % (unittest.F2B.fast, unittest.F2B.no_gamin))
 		from ..server.filtergamin import FilterGamin
 		filters.append(FilterGamin)
-	except ImportError, e: # pragma: no cover
+	except ImportError as e: # pragma: no cover
 		logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e)
 
 	try:
 		from ..server.filterpyinotify import FilterPyinotify
 		filters.append(FilterPyinotify)
-	except ImportError, e: # pragma: no cover
+	except ImportError as e: # pragma: no cover
 		logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e)
 
 	for Filter_ in filters:
@@ -448,7 +448,7 @@ def gatherTests(regexps=None, opts=None):
 	try: # pragma: systemd no cover
 		from ..server.filtersystemd import FilterSystemd
 		tests.addTest(unittest.makeSuite(filtertestcase.get_monitor_failures_journal_testcase(FilterSystemd)))
-	except ImportError, e: # pragma: no cover
+	except ImportError as e: # pragma: no cover
 		logSys.warning("I: Skipping systemd backend testing. Got exception '%s'" % e)
 
 	# Server test for logging elements which break logging used to support

From e3a75b4f1efc8186551527da368c454b5b2e9cb1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 6 Sep 2016 14:27:42 +0200
Subject: [PATCH 168/623] optimize test cases after merge (using 0.10-features)

---
 fail2ban/tests/filtertestcase.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index c04fa4ec..f7394a8e 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1145,6 +1145,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			# Now let's feed it with entries from the file
 			_copy_lines_to_journal(
 				self.test_file, self.journal_fields, skip=15, n=4)
+			self.waitForTicks(1)
 			self.assertTrue(self.isFilled(10))
 			self.assert_correct_ban("87.142.124.10", 4)
 			# Add direct utf, unicode, blob:
@@ -1159,10 +1160,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				fields = self.journal_fields
 				fields.update(TEST_JOURNAL_FIELDS)
 				journal.send(MESSAGE=l, **fields)
-			self.assertTrue(self.isFilled(10))
-			endtm = MyTime.time()+10
-			while len(self.jail) != 2 and MyTime.time() < endtm:
-				time.sleep(0.10)
+			self.waitFailTotal(6, 10)
+			self.assertTrue(Utils.wait_for(lambda: len(self.jail) == 2, 10))
 			self.assertEqual(sorted([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()]), 
 				["192.0.2.1", "192.0.2.2"])
 

From f512628af2cced65a6be469561a6e78c8f150aad Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 6 Sep 2016 20:15:45 +0200
Subject: [PATCH 169/623] make with_foreground_server_thread decorator to test
 several client/server commands

---
 fail2ban/tests/fail2banclienttestcase.py | 96 ++++++++++++++----------
 1 file changed, 55 insertions(+), 41 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index ed2d3c46..6d4691fe 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -41,6 +41,7 @@ from ..client.fail2banclient import exec_command_line as _exec_client, VisualWai
 from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server
 from .. import protocol
 from ..server import server
+from ..server.mytime import MyTime
 from ..server.utils import Utils
 from .utils import LogCaptureTestCase, with_tmpdir, shutil, logging
 
@@ -303,47 +304,60 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		phase['end'] = True
 		logSys.debug("end of test worker")
 
-	@with_tmpdir
-	def testStartForeground(self, tmp):
-		# intended to be ran only in subclasses
-		th = None
-		phase = dict()
-		try:
-			# started directly here, so prevent overwrite test cases logger with "INHERITED"
-			startparams = _start_params(tmp, logtarget="INHERITED")
-			# because foreground block execution - start it in thread:
-			th = Thread(
-				name="_TestCaseWorker",
-				target=self._testStartForeground,
-				args=(tmp, startparams, phase)
-			)
-			th.daemon = True
-			th.start()
-			try:
-				# wait for start thread:
-				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('start', None))
-				# wait for server (socket and ready):
-				self._wait_for_srv(tmp, True, startparams=startparams)
-				self.pruneLog()
-				# several commands to server:
-				self.execSuccess(startparams, "ping")
-				self.execFailed(startparams, "~~unknown~cmd~failed~~")
-				self.execSuccess(startparams, "echo", "TEST-ECHO")
-			finally:
-				self.pruneLog()
-				# stop:
-				self.execSuccess(startparams, "stop")
-				# wait for end:
-				Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-				self.assertTrue(phase.get('end', None))
-				self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-		finally:
-			if th:
-				# we start client/server directly in current process (new thread),
-				# so don't kill (same process) - if success, just wait for end of worker:
-				if phase.get('end', None):
-					th.join()
+	def with_foreground_server_thread(startextra={}):
+		"""Helper to decorate tests uses foreground server (as thread), started directly in test-cases
+
+		To be used only in subclasses
+		"""
+		def _deco_wrapper(f):
+			@with_tmpdir
+			@wraps(f)
+			def wrapper(self, tmp, *args, **kwargs):
+				th = None
+				phase = dict()
+				try:
+					# started directly here, so prevent overwrite test cases logger with "INHERITED"
+					startparams = _start_params(tmp, logtarget="INHERITED", **startextra)
+					# because foreground block execution - start it in thread:
+					th = Thread(
+						name="_TestCaseWorker",
+						target=self._testStartForeground,
+						args=(tmp, startparams, phase)
+					)
+					th.daemon = True
+					th.start()
+					try:
+						# wait for start thread:
+						Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+						self.assertTrue(phase.get('start', None))
+						# wait for server (socket and ready):
+						self._wait_for_srv(tmp, True, startparams=startparams)
+						self.pruneLog()
+						# several commands to server in body of decorated function:
+						return f(self, tmp, startparams, *args, **kwargs)
+					finally:
+						self.pruneLog()
+						# stop:
+						self.execSuccess(startparams, "stop")
+						# wait for end:
+						Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+						self.assertTrue(phase.get('end', None))
+						self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+				finally:
+					if th:
+						# we start client/server directly in current process (new thread),
+						# so don't kill (same process) - if success, just wait for end of worker:
+						if phase.get('end', None):
+							th.join()
+			return wrapper
+		return _deco_wrapper
+
+	@with_foreground_server_thread()
+	def testStartForeground(self, tmp, startparams):
+		# several commands to server:
+		self.execSuccess(startparams, "ping")
+		self.execFailed(startparams, "~~unknown~cmd~failed~~")
+		self.execSuccess(startparams, "echo", "TEST-ECHO")
 
 
 class Fail2banClientTest(Fail2banClientServerBase):

From b12a3acb06fed4f240e1cea20f4b07f913edf221 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 6 Sep 2016 23:54:19 +0200
Subject: [PATCH 170/623] temp commit: reload not ready...

---
 fail2ban/client/beautifier.py            |   2 +-
 fail2ban/client/fail2banclient.py        |  43 ++-
 fail2ban/protocol.py                     |   7 +-
 fail2ban/server/actions.py               |  29 +-
 fail2ban/server/filter.py                |  97 ++++--
 fail2ban/server/filterpoll.py            |  30 +-
 fail2ban/server/filtersystemd.py         |  10 +-
 fail2ban/server/jail.py                  |   3 +-
 fail2ban/server/jails.py                 |  11 +-
 fail2ban/server/server.py                | 134 ++++++--
 fail2ban/server/transmitter.py           |  36 +-
 fail2ban/tests/fail2banclienttestcase.py | 403 +++++++++++++++++++----
 fail2ban/tests/servertestcase.py         |   4 +-
 fail2ban/tests/utils.py                  |   8 +-
 14 files changed, 646 insertions(+), 171 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index bd803a6a..2d2c0caf 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -46,7 +46,7 @@ class Beautifier:
 		return self.__inputCmd
 
 	def beautify(self, response):
-		logSys.debug(
+		logSys.log(5,
 			"Beautify " + repr(response) + " with " + repr(self.__inputCmd))
 		inC = self.__inputCmd
 		msg = response
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 788906aa..9bc1525e 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -91,7 +91,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						client = CSocket(self._conf["socket"])
 					ret = client.send(c)
 					if ret[0] == 0:
-						logSys.debug("OK : %r", ret[1])
+						logSys.log(5, "OK : %r", ret[1])
 						if showRet or c[0] == 'echo':
 							output(beautifier.beautify(ret[1]))
 					else:
@@ -104,7 +104,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						if showRet or c != ["ping"]:
 							self.__logSocketError()
 						else:
-							logSys.debug(" -- ping failed -- %r", e)
+							logSys.log(5, " -- ping failed -- %r", e)
 					return False
 				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
@@ -226,11 +226,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# prepare: read config, check configuration is valid, etc.:
 		if phase is not None:
 			phase['start'] = True
-			logSys.debug('  client phase %s', phase)
+			logSys.log(5, '  client phase %s', phase)
 		stream = self.__prepareStartServer()
 		if phase is not None:
 			phase['ready'] = phase['start'] = (True if stream else False)
-			logSys.debug('  client phase %s', phase)
+			logSys.log(5, '  client phase %s', phase)
 		if not stream:
 			return False
 		# configure server with config stream:
@@ -246,6 +246,10 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 	# @param cmd the command line
 
 	def __processCommand(self, cmd):
+		# wrap tuple to list (because could be modified here):
+		if not isinstance(cmd, list):
+			cmd = list(cmd)
+		# process:
 		if len(cmd) == 1 and cmd[0] == "start":
 
 			ret = self.__startServer(self._conf["background"])
@@ -253,8 +257,12 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				return False
 			return ret
 
-		elif len(cmd) == 1 and cmd[0] == "restart":
-
+		elif len(cmd) >= 1 and cmd[0] == "restart":
+			# if restart jail - re-operate via "reload --restart ...":
+			if len(cmd) > 1:
+				cmd[0:1] = ["reload", "--restart"]
+				return self.__processCommand(cmd)
+			# restart server:
 			if self._conf.get("interactive", False):
 				output('  ## stop ... ')
 			self.__processCommand(['stop'])
@@ -273,9 +281,21 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			return self.__processCommand(['start'])
 
 		elif len(cmd) >= 1 and cmd[0] == "reload":
+			# reload options:
+			opts = []
+			while len(cmd) >= 2:
+				if cmd[1] in ('--restart', "--unban", "--if-exists"):
+					opts.append(cmd[1])
+					del cmd[1]
+				else:
+					if len(cmd) > 2:
+						logSys.error("Unexpected argument(s) for reload: %r", cmd[1:])
+						return False
+					# stop options - jail name or --all
+					break
 			if self.__ping():
 				if len(cmd) == 1:
-					jail = 'all'
+					jail = '--all'
 					ret, stream = self.readConfig()
 				else:
 					jail = cmd[1]
@@ -283,9 +303,10 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				# Do not continue if configuration is not 100% valid
 				if not ret:
 					return False
-				self.__processCmd([['stop', jail]], False)
-				# Configure the server
-				return self.__processCmd(stream, True)
+				if self._conf.get("interactive", False):
+					output('  ## reload ... ')
+				# Reconfigure the server
+				return self.__processCmd([['reload', jail, opts, stream]], True)
 			else:
 				logSys.error("Could not find server")
 				return False
@@ -320,7 +341,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			maxtime = self._conf["timeout"]
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
-		logSys.debug("__waitOnServer: %r", (alive, maxtime))
+		logSys.log(5, "__waitOnServer: %r", (alive, maxtime))
 		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping()
 		with VisualWait(self._conf["verbose"]) as vis:
 			sltime = 0.0125 / 2
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index d671f3c3..147cd38d 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -49,8 +49,11 @@ protocol = [
 ['', "BASIC", ""],
 ["start", "starts the server and the jails"], 
 ["restart", "restarts the server"], 
-["reload", "reloads the configuration without restart"], 
-["reload <JAIL>", "reloads the jail <JAIL>"], 
+["restart [--unban] [--if-exists] <JAIL>", "restarts the jail <JAIL> (alias for 'reload --restart ... <JAIL>')"], 
+["reload [--restart] [--unban] [--all]", "reloads the configuration without restarting of the server, the option '--restart' activates completely restarting of affected jails, thereby unbans IP addresses (if option '--unban' specified)"],
+["reload [--restart] [--unban] [--if-exists] <JAIL>", "reloads the jail <JAIL>, or restarts it (if option '--restart' specified)"],
+["unban --all", "unbans all IP addresses (in all jails)"],
+["unban <IP> ... <IP>", "unbans <IP> (in all jails)"],
 ["stop", "stops all jails and terminate the server"], 
 ["status", "gets the current status of the server"], 
 ["ping", "tests if the server is alive"],
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 050c12e6..2fab02cd 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -180,7 +180,7 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
-	def removeBannedIP(self, ip):
+	def removeBannedIP(self, ip=None, db=True, ifexists=False):
 		"""Removes banned IP calling actions' unban method
 
 		Remove a banned IP now, rather than waiting for it to expire,
@@ -188,16 +188,20 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ip : str or IPAddr
-			The IP address  to unban
+		ip : str or IPAddr or None
+			The IP address to unban or all IPs if None
 
 		Raises
 		------
 		ValueError
 			If `ip` is not banned
 		"""
+		# Unban all?
+		if ip is None:
+			return self.__flushBan(db)
+		# Single IP:
 		# Always delete ip from database (also if currently not banned)
-		if self._jail.database is not None:
+		if db and self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
 		# Find the ticket with the IP.
 		ticket = self.__banManager.getTicketByIP(ip)
@@ -205,7 +209,11 @@ class Actions(JailThread, Mapping):
 			# Unban the IP.
 			self.__unBan(ticket)
 		else:
-			raise ValueError("IP %s is not banned" % ip)
+			if ifexists:
+				return 0
+			raise ValueError("%s is not banned" % ip)
+		return 1
+
 
 	def run(self):
 		"""Main loop for Threading.
@@ -336,14 +344,21 @@ class Actions(JailThread, Mapping):
 		for ticket in self.__banManager.unBanList(MyTime.time()):
 			self.__unBan(ticket)
 
-	def __flushBan(self):
+	def __flushBan(self, db=False):
 		"""Flush the ban list.
 
 		Unban all IP address which are still in the banning list.
 		"""
 		logSys.debug("Flush ban list")
-		for ticket in self.__banManager.flushBanList():
+		lst = self.__banManager.flushBanList()
+		for ticket in lst:
+			# delete ip from database also:
+			if db and self._jail.database is not None:
+				ip = str(ticket.getIP())
+				self._jail.database.delBan(self._jail, ip)
+			# unban ip:
 			self.__unBan(ticket)
+		return len(lst)
 
 	def __unBan(self, ticket):
 		"""Unbans host corresponding to the ticket.
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e689ff00..4c67470e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -88,8 +88,9 @@ class Filter(JailThread):
 		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
 		self.__encoding = locale.getpreferredencoding()
-		## Error counter
-		self.__errors = 0
+		## Error counter (protected, so can be used in filter implementations)
+		## if it reached 100 (at once), run-cycle will go idle
+		self._errors = 0
 		## Ticks counter
 		self.ticks = 0
 
@@ -100,6 +101,30 @@ class Filter(JailThread):
 	def __repr__(self):
 		return "%s(%r)" % (self.__class__.__name__, self.jail)
 
+	def clearAllParams(self):
+		""" Clear all lists/dicts parameters (used by reloading)
+		"""
+		self.delFailRegex()
+		self.delIgnoreRegex()
+		self.delIgnoreIP()
+
+	def reload(self, begin=True):
+		""" Begin or end of reloading resp. refreshing of all parameters
+		"""
+		if begin:
+			self.clearAllParams()
+			if hasattr(self, 'getLogPaths'):
+				self._reload_logs = dict((k, 1) for k in self.getLogPaths())
+		else:
+			if hasattr(self, '_reload_logs'):
+				dellogs = dict()
+				# if it was not reloaded - remove obsolete log file:
+				for path in self._reload_logs:
+					self.delLogPath(path)
+				delattr(self, '_reload_logs')
+
+
+
 	##
 	# Add a regular expression which matches the failure.
 	#
@@ -119,8 +144,13 @@ class Filter(JailThread):
 			logSys.error(e)
 			raise e
 
-	def delFailRegex(self, index):
+	def delFailRegex(self, index=None):
 		try:
+			# clear all:
+			if index is None:
+				del self.__failRegex[:]
+				return
+			# delete by index:
 			del self.__failRegex[index]
 		except IndexError:
 			logSys.error("Cannot remove regular expression. Index %d is not "
@@ -152,8 +182,13 @@ class Filter(JailThread):
 			logSys.error(e)
 			raise e 
 
-	def delIgnoreRegex(self, index):
+	def delIgnoreRegex(self, index=None):
 		try:
+			# clear all:
+			if index is None:
+				del self.__ignoreRegex[:]
+				return
+			# delete by index:
 			del self.__ignoreRegex[index]
 		except IndexError:
 			logSys.error("Cannot remove regular expression. Index %d is not "
@@ -203,7 +238,7 @@ class Filter(JailThread):
 		value = MyTime.str2seconds(value)
 		self.__findTime = value
 		self.failManager.setMaxTime(value)
-		logSys.info("Set findtime = %s" % value)
+		logSys.info("  findtime: %s", value)
 
 	##
 	# Get the time needed to find a failure.
@@ -232,10 +267,10 @@ class Filter(JailThread):
 			template = DatePatternRegex(pattern)
 		self.dateDetector = DateDetector()
 		self.dateDetector.appendTemplate(template)
-		logSys.info("Date pattern set to `%r`: `%s`" %
-			(pattern, template.name))
-		logSys.debug("Date pattern regex for %r: %s" %
-			(pattern, template.regex))
+		logSys.info("  date pattern `%r`: `%s`",
+			pattern, template.name)
+		logSys.debug("  date pattern regex for %r: %s",
+			pattern, template.regex)
 
 	##
 	# Get the date detector pattern, or Default Detectors if not changed
@@ -261,7 +296,7 @@ class Filter(JailThread):
 
 	def setMaxRetry(self, value):
 		self.failManager.setMaxRetry(value)
-		logSys.info("Set maxRetry = %s" % value)
+		logSys.info("  maxRetry: %s", value)
 
 	##
 	# Get the maximum retry value.
@@ -280,7 +315,7 @@ class Filter(JailThread):
 		if int(value) <= 0:
 			raise ValueError("maxlines must be integer greater than zero")
 		self.__lineBufferSize = int(value)
-		logSys.info("Set maxlines = %i" % self.__lineBufferSize)
+		logSys.info("  maxLines: %i", self.__lineBufferSize)
 
 	##
 	# Get the maximum line buffer size.
@@ -300,7 +335,7 @@ class Filter(JailThread):
 			encoding = locale.getpreferredencoding()
 		codecs.lookup(encoding) # Raise LookupError if invalid codec
 		self.__encoding = encoding
-		logSys.info("Set jail log file encoding to %s" % encoding)
+		logSys.info("  encoding: %s" % encoding)
 		return encoding
 
 	##
@@ -375,11 +410,16 @@ class Filter(JailThread):
 		ip = IPAddr(ipstr)
 
 		# log and append to ignore list
-		logSys.debug("Add %r to ignore list (%r)", ip, ipstr)
+		logSys.debug("  Add %r to ignore list (%r)", ip, ipstr)
 		self.__ignoreIpList.append(ip)
 
-	def delIgnoreIP(self, ip):
-		logSys.debug("Remove %r from ignore list", ip)
+	def delIgnoreIP(self, ip=None):
+		# clear all:
+		if ip is None:
+			del self.__ignoreIpList[:]
+			return
+		# delete by ip:
+		logSys.debug("  Remove %r from ignore list", ip)
 		self.__ignoreIpList.remove(ip)
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
@@ -483,18 +523,18 @@ class Filter(JailThread):
 				tick = FailTicket(ip, unixTime, lines, data=fail)
 				self.failManager.addFailure(tick)
 			# reset (halve) error counter (successfully processed line):
-			if self.__errors:
-				self.__errors //= 2
+			if self._errors:
+				self._errors //= 2
 		except Exception as e:
 			logSys.error("Failed to process line: %r, caught exception: %r", line, e,
 				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			# incr error counter, stop processing (going idle) after 100th error :
-			self.__errors += 1
+			self._errors += 1
 			# sleep a little bit (to get around time-related errors):
 			time.sleep(self.sleeptime)
-			if self.__errors >= 100:
-				logSys.error("Too many errors at once (%s), going idle", self.__errors)
-				self.__errors //= 2
+			if self._errors >= 100:
+				logSys.error("Too many errors at once (%s), going idle", self._errors)
+				self._errors //= 2
 				self.idle = True
 
 	##
@@ -657,7 +697,10 @@ class FileFilter(Filter):
 
 	def addLogPath(self, path, tail=False, autoSeek=True):
 		if path in self.__logs:
-			logSys.error(path + " already exists")
+			if hasattr(self, '_reload_logs') and path in self._reload_logs:
+				del self._reload_logs[path]
+			else:
+				logSys.error(path + " already exists")
 		else:
 			log = FileContainer(path, self.getLogEncoding(), tail)
 			db = self.jail.database
@@ -666,7 +709,7 @@ class FileFilter(Filter):
 				if lastpos and not tail:
 					log.setPos(lastpos)
 			self.__logs[path] = log
-			logSys.info("Added logfile = %s (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
+			logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
 			if autoSeek:
 				# if default, seek to "current time" - "find time":
 				if isinstance(autoSeek, bool):
@@ -692,7 +735,7 @@ class FileFilter(Filter):
 		db = self.jail.database
 		if db is not None:
 			db.updateLog(self.jail, log)
-		logSys.info("Removed logfile = %s" % path)
+		logSys.info("Removed logfile: %r" % path)
 		self._delLogPath(path)
 		return
 
@@ -1055,10 +1098,14 @@ _decode_line_warn = {}
 
 class JournalFilter(Filter): # pragma: systemd no cover
 
+	def clearAllParams(self):
+		super(JournalFilter, self).clearAllParams()
+		self.delJournalMatch()
+
 	def addJournalMatch(self, match): # pragma: no cover - Base class, not used
 		pass
 
-	def delJournalMatch(self, match): # pragma: no cover - Base class, not used
+	def delJournalMatch(self, match=None): # pragma: no cover - Base class, not used
 		pass
 
 	def getJournalMatch(self, match): # pragma: no cover - Base class, not used
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 336807d4..f66be42b 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -31,7 +31,7 @@ from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger
+from ..helpers import getLogger, logging
 
 
 # Gets the instance of the logger.
@@ -137,28 +137,34 @@ class FilterPoll(FileFilter):
 		try:
 			logStats = os.stat(filename)
 			stats = logStats.st_mtime, logStats.st_ino, logStats.st_size
-			pstats = self.__prevStats.get(filename, ())
-			self.__file404Cnt[filename] = 0
+			pstats = self.__prevStats.get(filename, (0))
 			if logSys.getEffectiveLevel() <= 7:
 				# we do not want to waste time on strftime etc if not necessary
 				dt = logStats.st_mtime - pstats[0]
 				logSys.log(7, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s",
 				           filename, pstats, stats, dt)
 				# os.system("stat %s | grep Modify" % filename)
+			self.__file404Cnt[filename] = 0
 			if pstats == stats:
 				return False
 			logSys.debug("%s has been modified", filename)
 			self.__prevStats[filename] = stats
 			return True
-		except OSError as e:
-			logSys.error("Unable to get stat on %s because of: %s"
-						 % (filename, e))
+		except Exception as e:
+			# stil alive (may be deleted because multi-threaded):
+			if not self.getLog(filename):
+				logSys.warning("Log %r seems to be down: %s", filename, e)
+				return
+			# log error:
+			if self.__file404Cnt[filename] < 2:
+				logSys.error("Unable to get stat on %s because of: %s",
+							 filename, e, 
+							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
-			if self.__file404Cnt[filename] > 2:
-				logSys.warning("Too many errors. Setting the jail idle")
-				if self.jail is not None:
-					self.jail.idle = True
-				else:
-					logSys.warning("No jail is assigned to %s" % self)
+			self._errors += 1
+			if self.__file404Cnt[filename] > 50:
+				logSys.warning("Too many errors. Remove file %r from monitoring process", filename)
 				self.__file404Cnt[filename] = 0
+				self.delLogPath(filename)
 			return False
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 487af793..0b6c4074 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -154,12 +154,16 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	#
 	# @param match journalctl syntax matches
 
-	def delJournalMatch(self, match):
-		if match in self.__matches:
+	def delJournalMatch(self, match=None):
+		# clear all:
+		if match is None:
+			del self.__matches[:]
+		# delete by index:
+		elif match in self.__matches:
 			del self.__matches[self.__matches.index(match)]
-			self.resetJournalMatches()
 		else:
 			raise ValueError("Match not found")
+		self.resetJournalMatches()
 		logSys.info("Removed journal match for: %r" % " ".join(match))
 
 	##
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index 92f8190e..a5dc1119 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -79,6 +79,7 @@ class Jail(object):
 		logSys.info("Creating new jail '%s'" % self.name)
 		if backend is not None:
 			self._setBackend(backend)
+		self.backend = backend
 
 	def __repr__(self):
 		return "%s(%r)" % (self.__class__.__name__, self.name)
@@ -220,7 +221,7 @@ class Jail(object):
 				jail=self, bantime=self.actions.getBanTime()):
 				if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
 					self.__queue.put(ticket)
-		logSys.info("Jail '%s' started" % self.name)
+		logSys.info("Jail %r started", self.name)
 
 	def stop(self):
 		"""Stop the jail, by stopping filter and actions threads.
diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py
index 675b6276..972a8c4b 100644
--- a/fail2ban/server/jails.py
+++ b/fail2ban/server/jails.py
@@ -62,14 +62,15 @@ class Jails(Mapping):
 		DuplicateJailException
 			If jail name is already present.
 		"""
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			if name in self._jails:
-				raise DuplicateJailException(name)
+				if noduplicates:
+					raise DuplicateJailException(name)
 			else:
 				self._jails[name] = Jail(name, backend, db)
-		finally:
-			self.__lock.release()
+
+	def exists(self, name):
+		return name in self._jails
 
 	def __getitem__(self, name):
 		try:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 51487ed7..041b0983 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -67,6 +67,7 @@ class Server:
 		self.__db = None
 		self.__daemon = daemon
 		self.__transm = Transmitter(self)
+		self.__reload_state = {}
 		#self.__asyncServer = AsyncServer(self.__transm)
 		self.__asyncServer = None
 		self.__logLevel = None
@@ -184,41 +185,105 @@ class Server:
 		self.quit = lambda: False
 
 	def addJail(self, name, backend):
-		self.__jails.add(name, backend, self.__db)
+		addflg = True
+		if self.__reload_state.get(name) and self.__jails.exists(name):
+			jail = self.__jails[name]
+			# if backend switch - restart instead of reload:
+			if jail.backend == backend:
+				addflg = False
+				logSys.info("Reload jail %r", name)
+				# prevent to reload the same jail twice (temporary keep it in state, needed to commit reload):
+				self.__reload_state[name] = None
+			else:
+				logSys.info("Restart jail %r (reason: %r != %r)", name, jail.backend, backend)
+				self.delJail(name, stop=True)
+				# prevent to start the same jail twice (no reload more - restart):
+				del self.__reload_state[name]
+		if addflg:
+			self.__jails.add(name, backend, self.__db)
 		if self.__db is not None:
 			self.__db.addJail(self.__jails[name])
 		
-	def delJail(self, name):
+	def delJail(self, name, stop=True):
+		jail = self.__jails[name]
+		if stop and jail.isAlive():
+			logSys.debug("Stopping jail %r" % name)
+			jail.stop()
 		if self.__db is not None:
-			self.__db.delJail(self.__jails[name])
+			self.__db.delJail(jail)
 		del self.__jails[name]
 
 	def startJail(self, name):
-		try:
-			self.__lock.acquire()
-			if not self.__jails[name].isAlive():
-				self.__jails[name].start()
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			jail = self.__jails[name]
+			if not jail.isAlive():
+				jail.start()
+			elif name in self.__reload_state:
+				logSys.info("Jail %r reloaded", name)
+				del self.__reload_state[name]
+			if jail.idle:
+				jail.idle = False
 	
 	def stopJail(self, name):
-		logSys.debug("Stopping jail %s" % name)
-		try:
-			self.__lock.acquire()
-			if self.__jails[name].isAlive():
-				self.__jails[name].stop()
-				self.delJail(name)
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			self.delJail(name, stop=True)
 	
 	def stopAllJail(self):
 		logSys.info("Stopping all jails")
-		try:
-			self.__lock.acquire()
-			for jail in self.__jails.keys():
-				self.stopJail(jail)
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			for name in self.__jails.keys():
+				self.delJail(name, stop=True)
+
+	def reloadJails(self, name, opts, begin):
+		if begin:
+			# begin reload:
+			if self.__reload_state and (name == '--all' or self.__reload_state.get(name)):
+				raise ValueError('Reload already in progress')
+			logSys.info("Reload " + (("jail %s" % name) if name != '--all' else "all jails"))
+			with self.__lock:
+				# if single jail:
+				if name != '--all':
+					jail = None
+					# test jail exists (throws exception if not):
+					if "--if-exists" not in opts or self.__jails.exists(name):
+						jail = self.__jails[name]
+					if jail:
+						# first unban all ips (will be not restored after (re)start):
+						if "--unban" in opts:
+							self.setUnbanIP(name)
+						# stop if expected:
+						if "--restart" in opts:
+							self.stopJail(name)
+				else:
+					# first unban all ips (will be not restored after (re)start):
+					if "--unban" in opts:
+						self.setUnbanIP()
+					# stop if expected:
+					if "--restart" in opts:
+						self.stopAllJail()
+				# first set all affected jail(s) to idle and reset filter regex and other lists/dicts:
+				for jn, jail in self.__jails.iteritems():
+					if name == '--all' or jn == name:
+						jail.idle = True
+						self.__reload_state[jn] = jail
+						jail.filter.reload(begin=True)
+				pass
+		else:
+			# end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started:
+			with self.__lock:
+				deljails = []
+				for jn, jail in self.__jails.iteritems():
+					# still in reload state:
+					if jn in self.__reload_state:
+						# remove jails that are not reloaded (untouched, so not in new configuration)
+						deljails.append(jn)
+					else:
+						# commit (reload was finished):
+						jail.filter.reload(begin=False)
+				for jn in deljails:
+					self.delJail(jn)
+			self.__reload_state = {}
+			logSys.info("Reload finished.")
 
 	def setIdleJail(self, name, value):
 		self.__jails[name].idle = value
@@ -309,7 +374,7 @@ class Server:
 			logSys.debug("  failregex: %r", value)
 			flt.addFailRegex(value)
 	
-	def delFailRegex(self, name, index):
+	def delFailRegex(self, name, index=None):
 		self.__jails[name].filter.delFailRegex(index)
 	
 	def getFailRegex(self, name):
@@ -368,8 +433,20 @@ class Server:
 	def setBanIP(self, name, value):
 		return self.__jails[name].filter.addBannedIP(value)
 		
-	def setUnbanIP(self, name, value):
-		self.__jails[name].actions.removeBannedIP(value)
+	def setUnbanIP(self, name=None, value=None):
+		if name is not None:
+			# in all jails:
+			jails = [self.__jails[name]]
+		else:
+			# single jail:
+			jails = self.__jails.values()
+		# unban given or all (if value is None):
+		cnt = 0
+		for jail in jails:
+			cnt += jail.actions.removeBannedIP(value, ifexists=(name is None))
+		if value and not cnt:
+			logSys.info("%s is not banned", value)
+		return cnt
 		
 	def getBanTime(self, name):
 		return self.__jails[name].actions.getBanTime()
@@ -420,7 +497,10 @@ class Server:
 			if self.__logLevel == value:
 				return
 			try:
-				getLogger("fail2ban").setLevel(getattr(logging, value))
+				ll = getattr(logging, value)
+				# don't change real log-level if running from the test cases:
+				getLogger("fail2ban").setLevel(
+					ll if DEF_LOGTARGET != "INHERITED" or ll < logging.DEBUG else DEF_LOGLEVEL)
 				self.__logLevel = value
 			except AttributeError:
 				raise ValueError("Invalid log level %r" % value)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 698dad34..19cf51ef 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -27,7 +27,7 @@ __license__ = "GPL"
 import time
 import json
 
-from ..helpers import getLogger
+from ..helpers import getLogger, logging
 from .. import version
 
 # Gets the instance of the logger.
@@ -52,13 +52,14 @@ class Transmitter:
 	
 	def proceed(self, command):
 		# Deserialize object
-		logSys.debug("Command: %r", command)
+		logSys.log(5, "Command: %r", command)
 		try:
 			ret = self.__commandHandler(command)
 			ack = 0, ret
 		except Exception as e:
-			logSys.warning("Command %r has failed. Received %r"
-						% (command, e))
+			logSys.warning("Command %r has failed. Received %r",
+						command, e, 
+						exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			ack = 1, e
 		return ack
 	
@@ -72,8 +73,8 @@ class Transmitter:
 			return "pong"
 		elif command[0] == "add":
 			name = command[1]
-			if name == "all":
-				raise Exception("Reserved name")
+			if name == "--all":
+				raise Exception("Reserved name %r" % (name,))
 			try:
 				backend = command[2]
 			except IndexError:
@@ -87,12 +88,31 @@ class Transmitter:
 		elif command[0] == "stop":
 			if len(command) == 1:
 				self.__server.quit()
-			elif command[1] == "all":
+			elif command[1] == "--all":
 				self.__server.stopAllJail()
 			else:
 				name = command[1]
 				self.__server.stopJail(name)
 			return None
+		elif command[0] == "reload":
+			opts = command[1:3]
+			try:
+				self.__server.reloadJails(*opts, begin=True)
+				for cmd in command[3]:
+					self.__commandHandler(cmd)
+			finally:
+				self.__server.reloadJails(*opts, begin=False)
+			return None
+		elif len(command) >= 2 and command[0] == "unban":
+			# unban in all jails:
+			value = command[1:]
+			# if all ips:
+			if len(value) == 1 and value[0] == "--all":
+				self.__server.setUnbanIP()
+				return
+			for value in value:
+				self.__server.setUnbanIP(None, value)
+			return None
 		elif command[0] == "echo":
 			return command[1:]
 		elif command[0] == "sleep":
@@ -265,7 +285,7 @@ class Transmitter:
 			action = self.__server.getAction(name, actionname)
 			if multiple:
 				for cmd in command[3]:
-					logSys.debug("  %r", cmd)
+					logSys.log(5, "  %r", cmd)
 					actionkey = cmd[0]
 					if callable(getattr(action, actionkey, None)):
 						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 6d4691fe..557f0ada 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -43,7 +43,7 @@ from .. import protocol
 from ..server import server
 from ..server.mytime import MyTime
 from ..server.utils import Utils
-from .utils import LogCaptureTestCase, with_tmpdir, shutil, logging
+from .utils import LogCaptureTestCase, logSys as DefLogSys, with_tmpdir, shutil, logging
 
 from ..helpers import getLogger
 
@@ -69,7 +69,8 @@ fail2bancmdline.logSys = \
 fail2banclient.logSys = \
 fail2banserver.logSys = logSys
 
-server.DEF_LOGTARGET = "/dev/null"
+SRV_DEF_LOGTARGET = server.DEF_LOGTARGET
+SRV_DEF_LOGLEVEL = server.DEF_LOGLEVEL
 
 def _test_output(*args):
 	logSys.info(args[0])
@@ -111,17 +112,25 @@ fail2bancmdline.PRODUCTION = \
 fail2banserver.PRODUCTION = False
 
 
-def _out_file(fn):
+def _out_file(fn, handle=logSys.debug):
 	"""Helper which outputs content of the file at HEAVYDEBUG loglevels"""
-	logSys.debug('---- ' + fn + ' ----')
+	handle('---- ' + fn + ' ----')
 	for line in fileinput.input(fn):
 		line = line.rstrip('\n')
-		logSys.debug(line)
-	logSys.debug('-'*30)
+		handle(line)
+	handle('-'*30)
 
 
-def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
+def _write_file(fn, mode, *lines):
+	f = open(fn, mode)
+	f.write('\n'.join(lines))
+	f.close()
+
+
+def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 	cfg = pjoin(tmp, "config")
+	if db == 'auto':
+		db = pjoin(tmp, "f2b-db.sqlite3")
 	if use_stock and STOCK:
 		# copy config (sub-directories as alias):
 		def ig_dirs(dir, files):
@@ -147,8 +156,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 	else:
 		# just empty config directory without anything (only fail2ban.conf/jail.conf):
 		os.mkdir(cfg)
-		f = open(pjoin(cfg, "fail2ban.conf"), "w")
-		f.write('\n'.join((
+		_write_file(pjoin(cfg, "fail2ban.conf"), "w",
 			"[Definition]",
 			"loglevel = INFO",
 			"logtarget = " + logtarget,
@@ -156,19 +164,16 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"):
 			"socket = " + pjoin(tmp, "f2b.sock"),
 			"pidfile = " + pjoin(tmp, "f2b.pid"),
 			"backend = polling",
-			"dbfile = :memory:",
+			"dbfile = " + db,
 			"dbpurgeage = 1d",
 			"",
-		)))
-		f.close()
-		f = open(pjoin(cfg, "jail.conf"), "w")
-		f.write('\n'.join((
+		)
+		_write_file(pjoin(cfg, "jail.conf"), "w",
 			"[INCLUDES]", "",
 			"[DEFAULT]", "",
 			"",
-		)))
-		f.close()
-		if logSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		)
+		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(pjoin(cfg, "fail2ban.conf"))
 			_out_file(pjoin(cfg, "jail.conf"))
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
@@ -238,19 +243,78 @@ def with_kill_srv(f):
 			_kill_srv(pidfile)
 	return wrapper
 
+def with_foreground_server_thread(startextra={}):
+	"""Helper to decorate tests uses foreground server (as thread), started directly in test-cases
+
+	To be used only in subclasses
+	"""
+	def _deco_wrapper(f):
+		@with_tmpdir
+		@wraps(f)
+		def wrapper(self, tmp, *args, **kwargs):
+			th = None
+			phase = dict()
+			try:
+				# started directly here, so prevent overwrite test cases logger with "INHERITED"
+				startparams = _start_params(tmp, logtarget="INHERITED", **startextra)
+				# because foreground block execution - start it in thread:
+				th = Thread(
+					name="_TestCaseWorker",
+					target=self._testStartForeground,
+					args=(tmp, startparams, phase)
+				)
+				th.daemon = True
+				th.start()
+				try:
+					# wait for start thread:
+					Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+					self.assertTrue(phase.get('start', None))
+					# wait for server (socket and ready):
+					self._wait_for_srv(tmp, True, startparams=startparams)
+					DefLogSys.info('=== within server: begin ===')
+					self.pruneLog()
+					# several commands to server in body of decorated function:
+					return f(self, tmp, startparams, *args, **kwargs)
+				finally:
+					DefLogSys.info('=== within server: end.  ===')
+					self.pruneLog()
+					# stop:
+					self.execSuccess(startparams, "stop")
+					# wait for end:
+					Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+					self.assertTrue(phase.get('end', None))
+					self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+			finally:
+				if th:
+					# we start client/server directly in current process (new thread),
+					# so don't kill (same process) - if success, just wait for end of worker:
+					if phase.get('end', None):
+						th.join()
+		return wrapper
+	return _deco_wrapper
+
 
 class Fail2banClientServerBase(LogCaptureTestCase):
 
 	_orig_exit = Fail2banCmdLine._exit
 
+	def _setLogLevel(self, *args, **kwargs):
+		pass
+
 	def setUp(self):
 		"""Call before every test case."""
 		LogCaptureTestCase.setUp(self)
+		# prevent to switch the logging in the test cases (use inherited one):
+		server.DEF_LOGTARGET = "INHERITED"
+		server.DEF_LOGLEVEL = DefLogSys.level
 		Fail2banCmdLine._exit = staticmethod(self._test_exit)
 
 	def tearDown(self):
 		"""Call after every test case."""
 		Fail2banCmdLine._exit = self._orig_exit
+		# restore server log target:
+		server.DEF_LOGTARGET = SRV_DEF_LOGTARGET
+		server.DEF_LOGLEVEL = SRV_DEF_LOGLEVEL
 		LogCaptureTestCase.tearDown(self)
 
 	@staticmethod
@@ -304,54 +368,6 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		phase['end'] = True
 		logSys.debug("end of test worker")
 
-	def with_foreground_server_thread(startextra={}):
-		"""Helper to decorate tests uses foreground server (as thread), started directly in test-cases
-
-		To be used only in subclasses
-		"""
-		def _deco_wrapper(f):
-			@with_tmpdir
-			@wraps(f)
-			def wrapper(self, tmp, *args, **kwargs):
-				th = None
-				phase = dict()
-				try:
-					# started directly here, so prevent overwrite test cases logger with "INHERITED"
-					startparams = _start_params(tmp, logtarget="INHERITED", **startextra)
-					# because foreground block execution - start it in thread:
-					th = Thread(
-						name="_TestCaseWorker",
-						target=self._testStartForeground,
-						args=(tmp, startparams, phase)
-					)
-					th.daemon = True
-					th.start()
-					try:
-						# wait for start thread:
-						Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-						self.assertTrue(phase.get('start', None))
-						# wait for server (socket and ready):
-						self._wait_for_srv(tmp, True, startparams=startparams)
-						self.pruneLog()
-						# several commands to server in body of decorated function:
-						return f(self, tmp, startparams, *args, **kwargs)
-					finally:
-						self.pruneLog()
-						# stop:
-						self.execSuccess(startparams, "stop")
-						# wait for end:
-						Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-						self.assertTrue(phase.get('end', None))
-						self.assertLogged("Shutdown successful", "Exiting Fail2ban")
-				finally:
-					if th:
-						# we start client/server directly in current process (new thread),
-						# so don't kill (same process) - if success, just wait for end of worker:
-						if phase.get('end', None):
-							th.join()
-			return wrapper
-		return _deco_wrapper
-
 	@with_foreground_server_thread()
 	def testStartForeground(self, tmp, startparams):
 		# several commands to server:
@@ -522,6 +538,24 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.assertLogged("Usage: ")
 		self.pruneLog()
 
+	@with_tmpdir
+	def testClientFailCommands(self, tmp):
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
+
+		# not started:
+		self.execFailed(startparams,
+			"reload", "jail")
+		self.assertLogged("Could not find server")
+		self.pruneLog()
+
+		# unexpected arg:
+		self.execFailed(startparams,
+			"--async", "reload", "--xxx", "jail")
+		self.assertLogged("Unexpected argument(s) for reload:")
+		self.pruneLog()
+
+
 	def testVisualWait(self):
 		sleeptime = 0.035
 		for verbose in (2, 0):
@@ -626,3 +660,242 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# again:
 		self.assertTrue(_kill_srv(tmp))
 		self.assertLogged("cleanup: no pidfile for")
+
+	@with_foreground_server_thread(startextra={'db': 'auto'})
+	def testServerReloadTest(self, tmp, startparams):
+		"""Very complicated test-case, that expected running server (foreground in thread).
+
+		In this test-case, each phase is related from previous one, 
+		so it cannot be splitted in multiple test cases.
+
+		It uses file database (instead of :memory:), to restore bans and log-file positions,
+		after restart/reload between phases.
+		"""
+		cfg = pjoin(tmp, "config")
+		test1log = pjoin(tmp, "test1.log")
+		test2log = pjoin(tmp, "test2.log")
+		test3log = pjoin(tmp, "test3.log")
+
+		def _write_jail_cfg(enabled=[1, 2]):
+			_write_file(pjoin(cfg, "jail.conf"), "w",
+				"[INCLUDES]", "",
+				"[DEFAULT]", "",
+				"maxretry = 3",
+				"findtime = 10m",
+				"failregex = ^\s*failure (401|403) from <HOST>",
+				"",
+				"[test-jail1]", "backend = polling", "filter =", "action =",
+				"logpath = " + test1log,
+				"          " + test2log if 2 in enabled else "",
+				"          " + test3log if 2 in enabled else "",
+				"failregex = ^\s*failure (401|403) from <HOST>",
+				"            ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
+				"enabled = true" if 1 in enabled else "",
+				"",
+				"[test-jail2]", "backend = polling", "filter =", "action =",
+				"logpath = " + test2log,
+				"enabled = true" if 2 in enabled else "",
+			)
+			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+				_out_file(pjoin(cfg, "jail.conf"))
+
+		_write_jail_cfg(enabled=[1])
+		_write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3))
+		_write_file(test2log, "w")
+		_write_file(test3log, "w")
+		
+		# reload and wait for ban:
+		self.pruneLog("[test-phase 1]")
+		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			_out_file(test1log)
+		self.execSuccess(startparams, "reload")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MAX_WAITTIME / 5.0))
+		self.assertLogged("Added logfile: %r" % test1log)
+		
+		# enable both jails, 3 logs for jail1, etc...
+		# truncate test-log - we should not find unban/ban again by reload:
+		self.pruneLog("[test-phase 2a]")
+		_write_jail_cfg()
+		_write_file(test1log, "w+")
+		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			_out_file(test1log)
+		self.execSuccess(startparams, "reload")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+		# test not unbanned / banned again:
+		self.assertNotLogged(
+			"[test-jail1] Unban 192.0.2.1", 
+			"[test-jail1] Ban 192.0.2.1", all=True)
+		# test 2 new log files:
+		self.assertLogged(
+			"Added logfile: %r" % test2log, 
+			"Added logfile: %r" % test3log, all=True)
+		# test 1 new jail:
+		self.assertLogged(
+			"Creating new jail 'test-jail2'",
+			"Jail 'test-jail2' started", all=True)
+
+		# write new failures:
+		self.pruneLog("[test-phase 2b]")
+		_write_file(test2log, "w+", *(
+			(str(int(MyTime.time())) + "   error 403 from 192.0.2.2: test 2",) * 3 +
+		  (str(int(MyTime.time())) + "   error 403 from 192.0.2.3: test 2",) * 3 +
+		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3
+		))
+		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			_out_file(test2log)
+		# test all will be found in jail1 and one in jail2:
+		self.assertTrue(
+			Utils.wait_for(lambda: \
+				self._is_logged("[test-jail1] Ban 192.0.2.2") and 
+				self._is_logged("[test-jail1] Ban 192.0.2.3") and 
+				self._is_logged("[test-jail1] Ban 192.0.2.4") and 
+				self._is_logged("[test-jail2] Ban 192.0.2.4")
+				, MAX_WAITTIME / 5.0))
+		# test ips at all not visible for jail2:
+		self.assertNotLogged(
+			"[test-jail2] Found 192.0.2.2", 
+			"[test-jail2] Ban 192.0.2.2",
+			"[test-jail2] Found 192.0.2.3", 
+			"[test-jail2] Ban 192.0.2.3", all=True)
+
+		# rotate logs:
+		_write_file(test1log, "w+")
+		_write_file(test2log, "w+")
+
+		# restart jail without unban all:
+		self.pruneLog("[test-phase 2c]")
+		self.execSuccess(startparams,
+			"restart", "test-jail2")
+		self.assertTrue(
+			Utils.wait_for(lambda: \
+				self._is_logged("Jail 'test-jail2' started") and
+				self._is_logged("[test-jail2] Ban 192.0.2.4")
+				, MAX_WAITTIME / 5.0))
+		# stop/start and ban/unban:
+		self.assertLogged(
+			"Jail 'test-jail2' stopped",
+			"Jail 'test-jail2' started",
+			"[test-jail2] Unban 192.0.2.4",
+			"[test-jail2] Ban 192.0.2.4", all=True
+		)
+
+		# restart jail with unban all:
+		self.pruneLog("[test-phase 2d]")
+		self.execSuccess(startparams,
+			"restart", "--unban", "test-jail2")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Jail 'test-jail2' started"),
+				MAX_WAITTIME / 5.0))
+		self.assertLogged(
+			"Jail 'test-jail2' stopped",
+			"Jail 'test-jail2' started",
+			"[test-jail2] Unban 192.0.2.4", all=True
+		)
+		# no more ban (unbanned all):
+		self.assertNotLogged(
+			"[test-jail2] Ban 192.0.2.4", all=True
+		)
+
+		# reload jail1 without restart (without ban/unban):
+		self.pruneLog("[test-phase 3]")
+		self.execSuccess(startparams, "reload", "test-jail1")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+		self.assertLogged(
+			"Reload jail 'test-jail1'",
+			"Jail 'test-jail1' reloaded", all=True)
+		self.assertNotLogged(
+			"Reload jail 'test-jail2'",
+			"Jail 'test-jail2' reloaded",
+			"Jail 'test-jail1' started", all=True
+		)
+
+		# whole reload, but this time with jail1 only (jail2 should be stopped via configuration):
+		self.pruneLog("[test-phase 4]")
+		_write_jail_cfg(enabled=[1])
+		self.execSuccess(startparams, "reload")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+		# test both jails should be reloaded:
+		self.assertLogged(
+			"Reload jail 'test-jail1'")
+		# test jail2 goes down:
+		self.assertLogged(
+			"Stopping jail 'test-jail2'", 
+			"Jail 'test-jail2' stopped", all=True)
+		# test 2 log files removed:
+		self.assertLogged(
+			"Removed logfile: %r" % test2log, 
+			"Removed logfile: %r" % test3log, all=True)
+
+		# now write failures again and check already banned (jail1 was alive the whole time) and new bans occurred (jail1 was alive the whole time):
+		self.pruneLog("[test-phase 5]")
+		_write_file(test1log, "w+", *(
+			(str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 5",) * 3 + 
+			(str(int(MyTime.time())) + "   error 403 from 192.0.2.5: test 5",) * 3 +
+			(str(int(MyTime.time())) + " failure 401 from 192.0.2.6: test 5",) * 3
+		))
+		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			_out_file(test1log)
+		# test "failure" regexp still available:
+		self.assertTrue(
+			Utils.wait_for(lambda: \
+				self._is_logged("[test-jail1] 192.0.2.1 already banned") and
+				self._is_logged("[test-jail1] Ban 192.0.2.6")
+				, MAX_WAITTIME / 5.0))
+		self.assertLogged(
+			"[test-jail1] Found 192.0.2.1",
+			"[test-jail1] Found 192.0.2.6", all=True
+		)
+		# test "error" regexp no more available:
+		self.assertNotLogged("[test-jail1] Found 192.0.2.5")
+
+		# unban single ips:
+		self.pruneLog("[test-phase 6]")
+		self.execSuccess(startparams,
+			"--async", "unban", "192.0.2.5", "192.0.2.6")
+		self.assertLogged(
+			"192.0.2.5 is not banned",
+			"[test-jail1] Unban 192.0.2.6", all=True
+		)
+
+		# reload all (one jail) with unban all:
+		self.pruneLog("[test-phase 7]")
+		self.execSuccess(startparams,
+			"reload", "--unban")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+		# reloads unbanned all:
+		self.assertLogged(
+			"Jail 'test-jail1' reloaded",
+			"[test-jail1] Unban 192.0.2.1",
+			"[test-jail1] Unban 192.0.2.2",
+			"[test-jail1] Unban 192.0.2.3",
+			"[test-jail1] Unban 192.0.2.4", all=True
+		)
+		# no restart occurred, no more ban (unbanned all using option "--unban"):
+		self.assertNotLogged(
+			"Jail 'test-jail1' stopped",
+			"Jail 'test-jail1' started",
+			"[test-jail1] Ban 192.0.2.1",
+			"[test-jail1] Ban 192.0.2.2",
+			"[test-jail1] Ban 192.0.2.3",
+			"[test-jail1] Ban 192.0.2.4", all=True
+		)
+
+		# several small cases:
+		self.pruneLog("[test-phase end-1]")
+		# wrong jail (not-started):
+		self.execFailed(startparams,
+			"--async", "reload", "test-jail2")
+		self.assertLogged("the jail 'test-jail2' does not exist")
+		self.pruneLog()
+		# unavailable jail (but exit 0), using --if-exists option:
+		self.execSuccess(startparams,
+			"--async", "reload", "--if-exists", "test-jail2")
+		self.assertNotLogged(
+			"Creating new jail 'test-jail2'",
+			"Jail 'test-jail2' started", all=True)
+		self.pruneLog()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 12bbb2f7..3ca8049b 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -240,7 +240,7 @@ class Transmitter(TransmitterBase):
 			self.transm.proceed(["add", self.jailName, "polling"])[0], 1)
 		# All name is reserved
 		self.assertEqual(
-			self.transm.proceed(["add", "all", "polling"])[0], 1)
+			self.transm.proceed(["add", "--all", "polling"])[0], 1)
 
 	def testStartStopJail(self):
 		self.assertEqual(
@@ -267,7 +267,7 @@ class Transmitter(TransmitterBase):
 		self.assertTrue( Utils.wait_for(
 			lambda: self.server.isAlive(2) and not isinstance(self.transm.proceed(["status", self.jailName]), RuntimeError),
 			3) )
-		self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None))
+		self.assertEqual(self.transm.proceed(["stop", "--all"]), (0, None))
 		self.assertTrue( Utils.wait_for( lambda: not len(self.server._Server__jails), 3) )
 		self.assertNotIn(self.jailName, self.server._Server__jails)
 		self.assertNotIn("TestJail2", self.server._Server__jails)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index bdc1c189..6eaf560b 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -119,6 +119,7 @@ def getOptParser(doc=""):
 
 def initProcess(opts):
 	# Logger:
+	global logSys
 	logSys = getLogger("fail2ban")
 
 	# Numerical level of verbosity corresponding to a log "level"
@@ -577,7 +578,8 @@ class LogCaptureTestCase(unittest.TestCase):
 			print("")
 			logSys.handlers += self._old_handlers
 			logSys.debug('='*10 + ' %s ' + '='*20, self.id())
-		logSys.setLevel(logging.DEBUG)
+		else:
+			logSys.setLevel(logging.DEBUG)
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -638,8 +640,10 @@ class LogCaptureTestCase(unittest.TestCase):
 				if s_ in logged: # pragma: no cover
 					self.fail("%r was found in the log: ===\n%s===" % (s_, logged))
 
-	def pruneLog(self):
+	def pruneLog(self, logphase=None):
 		self._log.truncate(0)
+		if logphase:
+			logSys.debug('='*5 + ' %s ' + '='*5, logphase)
 
 	def getLog(self):
 		return self._log.getvalue()

From 8c26cada27b67eed8d7b946aaafc3469c129edb5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 7 Sep 2016 22:06:11 +0200
Subject: [PATCH 171/623] temp commit: partially cherry picked from
 ban-time-incr branch

---
 fail2ban/protocol.py                     |  4 +-
 fail2ban/server/actions.py               |  2 +-
 fail2ban/server/database.py              | 43 +++++++++++++++++
 fail2ban/server/jail.py                  | 24 +++++++---
 fail2ban/server/ticket.py                | 11 +++++
 fail2ban/tests/fail2banclienttestcase.py | 59 ++++++++++++++----------
 6 files changed, 108 insertions(+), 35 deletions(-)

diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 147cd38d..6299794e 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -52,9 +52,9 @@ protocol = [
 ["restart [--unban] [--if-exists] <JAIL>", "restarts the jail <JAIL> (alias for 'reload --restart ... <JAIL>')"], 
 ["reload [--restart] [--unban] [--all]", "reloads the configuration without restarting of the server, the option '--restart' activates completely restarting of affected jails, thereby unbans IP addresses (if option '--unban' specified)"],
 ["reload [--restart] [--unban] [--if-exists] <JAIL>", "reloads the jail <JAIL>, or restarts it (if option '--restart' specified)"],
-["unban --all", "unbans all IP addresses (in all jails)"],
-["unban <IP> ... <IP>", "unbans <IP> (in all jails)"],
 ["stop", "stops all jails and terminate the server"], 
+["unban --all", "unbans all IP addresses (in all jails and database)"],
+["unban <IP> ... <IP>", "unbans <IP> (in all jails and database)"],
 ["status", "gets the current status of the server"], 
 ["ping", "tests if the server is alive"],
 ["echo", "for internal usage, returns back and outputs a given string"],
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 2fab02cd..8c60e55f 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -320,7 +320,7 @@ class Actions(JailThread, Mapping):
 				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
 				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
 			if self.__banManager.addBanTicket(bTicket):
-				logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"]))
+				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.getRestored() else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
 						action.ban(aInfo.copy())
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 4e2dddf1..fc544be4 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -576,6 +576,49 @@ class Fail2BanDb(object):
 				self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 			return tickets if ip is None else ticket
 
+	@commitandrollback
+	def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None):
+		if fromtime is None:
+			fromtime = MyTime.time()
+		queryArgs = []
+		if jail is not None:
+			query = "SELECT ip, timeofban, data FROM bans WHERE jail=?"
+			queryArgs.append(jail.name)
+		else:
+			query = "SELECT ip, max(timeofban), data FROM bans WHERE 1"
+		if ip is not None:
+			query += " AND ip=?"
+			queryArgs.append(ip)
+		if forbantime is not None:
+			query += " AND timeofban > ?"
+			queryArgs.append(fromtime - forbantime)
+		if ip is None:
+			query += " GROUP BY ip ORDER BY ip, timeofban DESC"
+		cur = self._db.cursor()
+		return cur.execute(query, queryArgs)
+
+	def getCurrentBans(self, jail = None, ip = None, forbantime=None, fromtime=None):
+		tickets = []
+		ticket = None
+
+		results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
+
+		if results:
+			matches = []
+			failures = 0
+			for banip, timeofban, data in results:
+				#TODO: Implement data parts once arbitrary match keys completed
+				ticket = FailTicket(banip, timeofban, matches)
+				ticket.setAttempt(failures)
+				matches = []
+				failures = 0
+				matches.extend(data['matches'])
+				failures += data['failures']
+				ticket.setAttempt(failures)
+				tickets.append(ticket)
+
+		return tickets if ip is None else ticket
+
 	@commitandrollback
 	def purge(self, cur):
 		"""Purge old bans, jails and log files from database.
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index a5dc1119..7e05a025 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -194,7 +194,7 @@ class Jail(object):
 		Used by filter to add a failure for banning.
 		"""
 		self.__queue.put(ticket)
-		if self.database is not None:
+		if not ticket.getRestored() and self.database is not None:
 			self.database.addBan(self, ticket)
 
 	def getFailTicket(self):
@@ -207,6 +207,21 @@ class Jail(object):
 		except Queue.Empty:
 			return False
 
+	def restoreCurrentBans(self):
+		"""Restore any previous valid bans from the database.
+		"""
+		try:
+			if self.database is not None:
+				forbantime = self.actions.getBanTime()
+				for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime):
+					#logSys.debug('restored ticket: %s', ticket)
+					if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
+						# mark ticked was restored from database - does not put it again into db:
+						ticket.setRestored(True)
+						self.putFailTicket(ticket)
+		except Exception as e: # pragma: no cover
+			logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+
 	def start(self):
 		"""Start the jail, by starting filter and actions threads.
 
@@ -215,12 +230,7 @@ class Jail(object):
 		"""
 		self.filter.start()
 		self.actions.start()
-		# Restore any previous valid bans from the database
-		if self.database is not None:
-			for ticket in self.database.getBansMerged(
-				jail=self, bantime=self.actions.getBanTime()):
-				if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
-					self.__queue.put(ticket)
+		self.restoreCurrentBans()
 		logSys.info("Jail %r started", self.name)
 
 	def stop(self):
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 65ed83c3..8d07fcf3 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -36,6 +36,8 @@ logSys = getLogger(__name__)
 
 class Ticket:
 	
+	RESTORED = 0x01
+
 	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		"""Ticket constructor
 
@@ -126,6 +128,15 @@ class Ticket:
 	def getMatches(self):
 		return self._data.get('matches', [])
 
+	def setRestored(self, value):
+		if value:
+			self._flags = Ticket.RESTORED
+		else:
+			self._flags &= ~(Ticket.RESTORED)
+	
+	def getRestored(self):
+		return self._flags & Ticket.RESTORED
+
 	def setData(self, *args, **argv):
 		# if overwrite - set data and filter None values:
 		if len(args) == 1:
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 557f0ada..07bbd913 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -58,6 +58,7 @@ SERVER = "fail2ban-server"
 BIN = dirname(Fail2banServer.getServerPath())
 
 MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
+MID_WAITTIME = MAX_WAITTIME / 2.5
 
 ##
 # Several wrappers and settings for proper testing:
@@ -663,14 +664,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 	@with_foreground_server_thread(startextra={'db': 'auto'})
 	def testServerReloadTest(self, tmp, startparams):
-		"""Very complicated test-case, that expected running server (foreground in thread).
-
-		In this test-case, each phase is related from previous one, 
-		so it cannot be splitted in multiple test cases.
-
-		It uses file database (instead of :memory:), to restore bans and log-file positions,
-		after restart/reload between phases.
-		"""
+		# Very complicated test-case, that expected running server (foreground in thread).
+		#
+		# In this test-case, each phase is related from previous one, 
+		# so it cannot be splitted in multiple test cases.
+		# Additionaly many log-messages used as ready-sign (to wait for end of phase).
+		#
+		# Used file database (instead of :memory:), to restore bans and log-file positions,
+		# after restart/reload between phases.
 		cfg = pjoin(tmp, "config")
 		test1log = pjoin(tmp, "test1.log")
 		test2log = pjoin(tmp, "test2.log")
@@ -710,7 +711,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MID_WAITTIME))
 		self.assertLogged("Added logfile: %r" % test1log)
 		
 		# enable both jails, 3 logs for jail1, etc...
@@ -722,7 +723,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# test not unbanned / banned again:
 		self.assertNotLogged(
 			"[test-jail1] Unban 192.0.2.1", 
@@ -741,7 +742,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_file(test2log, "w+", *(
 			(str(int(MyTime.time())) + "   error 403 from 192.0.2.2: test 2",) * 3 +
 		  (str(int(MyTime.time())) + "   error 403 from 192.0.2.3: test 2",) * 3 +
-		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3
+		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 +
+		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.8: test 2",) * 3
 		))
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test2log)
@@ -751,8 +753,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				self._is_logged("[test-jail1] Ban 192.0.2.2") and 
 				self._is_logged("[test-jail1] Ban 192.0.2.3") and 
 				self._is_logged("[test-jail1] Ban 192.0.2.4") and 
-				self._is_logged("[test-jail2] Ban 192.0.2.4")
-				, MAX_WAITTIME / 5.0))
+				self._is_logged("[test-jail1] Ban 192.0.2.8") and 
+				self._is_logged("[test-jail2] Ban 192.0.2.4") and
+				self._is_logged("[test-jail2] Ban 192.0.2.8")
+				, MID_WAITTIME))
 		# test ips at all not visible for jail2:
 		self.assertNotLogged(
 			"[test-jail2] Found 192.0.2.2", 
@@ -771,14 +775,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertTrue(
 			Utils.wait_for(lambda: \
 				self._is_logged("Jail 'test-jail2' started") and
-				self._is_logged("[test-jail2] Ban 192.0.2.4")
-				, MAX_WAITTIME / 5.0))
-		# stop/start and ban/unban:
+				self._is_logged("[test-jail2] Restore Ban 192.0.2.4") and
+				self._is_logged("[test-jail2] Restore Ban 192.0.2.8")
+				, MID_WAITTIME))
+		# stop/start and unban/restore ban:
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
 			"[test-jail2] Unban 192.0.2.4",
-			"[test-jail2] Ban 192.0.2.4", all=True
+			"[test-jail2] Unban 192.0.2.8",
+			"[test-jail2] Restore Ban 192.0.2.4",
+			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
 
 		# restart jail with unban all:
@@ -787,22 +794,24 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"restart", "--unban", "test-jail2")
 		self.assertTrue(
 			Utils.wait_for(lambda: self._is_logged("Jail 'test-jail2' started"),
-				MAX_WAITTIME / 5.0))
+				MID_WAITTIME))
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
-			"[test-jail2] Unban 192.0.2.4", all=True
+			"[test-jail2] Unban 192.0.2.4",
+			"[test-jail2] Unban 192.0.2.8", all=True
 		)
 		# no more ban (unbanned all):
 		self.assertNotLogged(
-			"[test-jail2] Ban 192.0.2.4", all=True
+			"[test-jail2] Ban 192.0.2.4",
+			"[test-jail2] Ban 192.0.2.8", all=True
 		)
 
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
 		self.execSuccess(startparams, "reload", "test-jail1")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		self.assertLogged(
 			"Reload jail 'test-jail1'",
 			"Jail 'test-jail1' reloaded", all=True)
@@ -817,7 +826,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_jail_cfg(enabled=[1])
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# test both jails should be reloaded:
 		self.assertLogged(
 			"Reload jail 'test-jail1'")
@@ -844,7 +853,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			Utils.wait_for(lambda: \
 				self._is_logged("[test-jail1] 192.0.2.1 already banned") and
 				self._is_logged("[test-jail1] Ban 192.0.2.6")
-				, MAX_WAITTIME / 5.0))
+				, MID_WAITTIME))
 		self.assertLogged(
 			"[test-jail1] Found 192.0.2.1",
 			"[test-jail1] Found 192.0.2.6", all=True
@@ -866,7 +875,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.execSuccess(startparams,
 			"reload", "--unban")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# reloads unbanned all:
 		self.assertLogged(
 			"Jail 'test-jail1' reloaded",
@@ -885,7 +894,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail1] Ban 192.0.2.4", all=True
 		)
 
-		# several small cases:
+		# several small cases (cover several parts):
 		self.pruneLog("[test-phase end-1]")
 		# wrong jail (not-started):
 		self.execFailed(startparams,

From d2ddc59c40687e7846fd9aeb12230d9668dbf532 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 8 Sep 2016 15:38:36 +0200
Subject: [PATCH 172/623] build replacement of `<HOST>` substitution
 corresponding parameter `usedns` - dns-part will be added only if `usedns` is
 not `no`; new replacement for `<ADDR>` in opposition to `<HOST>`, for
 separate usage of 2 address groups only (regardless of `usedns`), `ip4` and
 `ip6` together, without host (dns)

---
 fail2ban/client/jailreader.py            |  2 +-
 fail2ban/server/failregex.py             | 23 +++++++++++++++--------
 fail2ban/server/filter.py                |  8 ++++----
 fail2ban/tests/fail2banclienttestcase.py |  3 ++-
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index e3c4731c..9d01a693 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -106,7 +106,7 @@ class JailReader(ConfigReader):
 				["int",    "maxretry", None],
 				["string", "findtime", None],
 				["string", "bantime", None],
-				["string", "usedns", None],
+				["string", "usedns", None], # be sure usedns is before all regex(s) in stream
 				["string", "failregex", None],
 				["string", "ignoreregex", None],
 				["string", "ignorecommand", None],
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 12b8986a..7c512609 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -40,11 +40,11 @@ class Regex:
 	# avoid construction of invalid object.
 	# @param value the regular expression
 	
-	def __init__(self, regex):
+	def __init__(self, regex, **kwargs):
 		self._matchCache = None
 		# Perform shortcuts expansions.
 		# Resolve "<HOST>" tag using default regular expression for host:
-		regex = Regex._resolveHostTag(regex)
+		regex = Regex._resolveHostTag(regex, **kwargs)
 		# Replace "<SKIPLINES>" with regular expression for multiple lines.
 		regexSplit = regex.split("<SKIPLINES>")
 		regex = regexSplit[0]
@@ -69,22 +69,29 @@ class Regex:
 	# @return the replaced regular expression as string
 
 	@staticmethod
-	def _resolveHostTag(regex):
-		# 3 groups instead of <HOST> - separated ipv4, ipv6 and host
-		regex = regex.replace("<HOST>",
-		 	r"""(?:(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<dns>[\w\-.^_]*\w))""")
+	def _resolveHostTag(regex, useDns="yes"):
 		# separated ipv4:
+		r_host = []
 		r = r"""(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})"""
 		regex = regex.replace("<IP4>", r); # self closed
 		regex = regex.replace("<F-IP4/>", r); # closed
+		r_host.append(r)
 		# separated ipv6:
 		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}?|(?<=:):))"""
 		regex = regex.replace("<IP6>", r); # self closed
 		regex = regex.replace("<F-IP6/>", r); # closed
+		r_host.append(r"""\[?%s\]?""" % (r,)); # enclose ipv6 in optional [] in host-regex
+		# 2 address groups instead of <ADDR> - in opposition to `<HOST>`, 
+		# for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together
+		regex = regex.replace("<ADDR>", "(?:%s)" % ("|".join(r_host),))
 		# separated dns:
 		r = r"""(?P<dns>[\w\-.^_]*\w)"""
 		regex = regex.replace("<DNS>", r); # self closed
 		regex = regex.replace("<F-DNS/>", r); # closed
+		if useDns not in ("no",):
+			r_host.append(r)
+		# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
+		regex = regex.replace("<HOST>", "(?:%s)" % ("|".join(r_host),))
 		# default failure-id as no space tag:
 		regex = regex.replace("<F-ID/>", r"""(?P<fid>\S+)"""); # closed
 		# default failure port, like 80 or http :
@@ -249,9 +256,9 @@ class FailRegex(Regex):
 	# avoid construction of invalid object.
 	# @param value the regular expression
 
-	def __init__(self, regex):
+	def __init__(self, regex, **kwargs):
 		# Initializes the parent.
-		Regex.__init__(self, regex)
+		Regex.__init__(self, regex, **kwargs)
 		# Check for group "dns", "ip4", "ip6", "fid"
 		if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]:
 			raise RegexException("No failure-id group in '%s'" % self._regex)
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 4c67470e..1c466c01 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -134,12 +134,12 @@ class Filter(JailThread):
 
 	def addFailRegex(self, value):
 		try:
-			regex = FailRegex(value)
+			regex = FailRegex(value, useDns=self.__useDns)
 			self.__failRegex.append(regex)
 			if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
 				logSys.warning(
-					"Mutliline regex set for jail '%s' "
-					"but maxlines not greater than 1")
+					"Mutliline regex set for jail %r "
+					"but maxlines not greater than 1", self.jail.name)
 		except RegexException as e:
 			logSys.error(e)
 			raise e
@@ -176,7 +176,7 @@ class Filter(JailThread):
 
 	def addIgnoreRegex(self, value):
 		try:
-			regex = Regex(value)
+			regex = Regex(value, useDns=self.__useDns)
 			self.__ignoreRegex.append(regex)
 		except RegexException as e:
 			logSys.error(e)
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 07bbd913..1ce35d92 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -58,7 +58,7 @@ SERVER = "fail2ban-server"
 BIN = dirname(Fail2banServer.getServerPath())
 
 MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
-MID_WAITTIME = MAX_WAITTIME / 2.5
+MID_WAITTIME = MAX_WAITTIME
 
 ##
 # Several wrappers and settings for proper testing:
@@ -681,6 +681,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_write_file(pjoin(cfg, "jail.conf"), "w",
 				"[INCLUDES]", "",
 				"[DEFAULT]", "",
+				"usedns = no",
 				"maxretry = 3",
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",

From 27f6fc083af5be470d49137e0cd4ecedcd54db52 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 8 Sep 2016 18:27:55 +0200
Subject: [PATCH 173/623] optimized BanManager: increase performance, fewer
 system load, try to prevent memory leakage: - better ban/unban handling
 within actions (e.g. used dict instead of list) - don't copy bans resp. its
 list on some operations; - added new unbantime handling to relieve unBanList
 (prevent permanent searching for tickets to unban) - prefer failure-ID as
 identifier of the ticket to its IP (most of the time the same, but it can be
 something else e.g. user name in some complex jails, as introduced in 0.10)

---
 fail2ban/server/actions.py           |  32 ++++--
 fail2ban/server/banmanager.py        | 152 +++++++++++++--------------
 fail2ban/server/ticket.py            |  16 ++-
 fail2ban/tests/banmanagertestcase.py |   2 +-
 4 files changed, 108 insertions(+), 94 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 8c60e55f..244d1e68 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -204,7 +204,7 @@ class Actions(JailThread, Mapping):
 		if db and self._jail.database is not None:
 			self._jail.database.delBan(self._jail, ip)
 		# Find the ticket with the IP.
-		ticket = self.__banManager.getTicketByIP(ip)
+		ticket = self.__banManager.getTicketByID(ip)
 		if ticket is not None:
 			# Unban the IP.
 			self.__unBan(ticket)
@@ -303,8 +303,11 @@ class Actions(JailThread, Mapping):
 		bool
 			True if an IP address get banned.
 		"""
-		ticket = self._jail.getFailTicket()
-		if ticket:
+		cnt = 0
+		while cnt < 100:
+			ticket = self._jail.getFailTicket()
+			if not ticket:
+				break
 			aInfo = CallingMap()
 			bTicket = BanManager.createBanTicket(ticket)
 			ip = bTicket.getIP()
@@ -320,6 +323,7 @@ class Actions(JailThread, Mapping):
 				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
 				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
 			if self.__banManager.addBanTicket(bTicket):
+				cnt += 1
 				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.getRestored() else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
@@ -330,19 +334,26 @@ class Actions(JailThread, Mapping):
 							"info '%r': %s",
 							self._jail.name, name, aInfo, e,
 							exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-				return True
 			else:
-				logSys.notice("[%s] %s already banned" % (self._jail.name,
-														aInfo["ip"]))
-		return False
+				logSys.notice("[%s] %s already banned", self._jail.name, ip)
+		if cnt:
+			logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, 
+				self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name)
+		return cnt
 
 	def __checkUnBan(self):
 		"""Check for IP address to unban.
 
 		Unban IP addresses which are outdated.
 		"""
-		for ticket in self.__banManager.unBanList(MyTime.time()):
+		lst = self.__banManager.unBanList(MyTime.time())
+		for ticket in lst:
 			self.__unBan(ticket)
+		cnt = len(lst)
+		if cnt:
+			logSys.debug("Unbanned %s, %s ticket(s) in %r", 
+				cnt, self.__banManager.size(), self._jail.name)
+		return cnt
 
 	def __flushBan(self, db=False):
 		"""Flush the ban list.
@@ -358,7 +369,10 @@ class Actions(JailThread, Mapping):
 				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
 			self.__unBan(ticket)
-		return len(lst)
+		cnt = len(lst)
+		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
+			cnt, self.__banManager.size(), self._jail.name)
+		return cnt
 
 	def __unBan(self, ticket):
 		"""Unbans host corresponding to the ticket.
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index afa70685..e0a9e5ca 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -51,11 +51,13 @@ class BanManager:
 		## Mutex used to protect the ban list.
 		self.__lock = Lock()
 		## The ban list.
-		self.__banList = list()
+		self.__banList = dict()
 		## The amount of time an IP address gets banned.
 		self.__banTime = 600
 		## Total number of banned IP address
 		self.__banTotal = 0
+		## The time for next unban process (for performance and load reasons):
+		self.__nextUnbanTime = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
 	
 	##
 	# Set the ban time.
@@ -64,11 +66,8 @@ class BanManager:
 	# @param value the time
 	
 	def setBanTime(self, value):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			self.__banTime = int(value)
-		finally:
-			self.__lock.release()
 	
 	##
 	# Get the ban time.
@@ -77,11 +76,8 @@ class BanManager:
 	# @return the time
 	
 	def getBanTime(self):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			return self.__banTime
-		finally:
-			self.__lock.release()
 	
 	##
 	# Set the total number of banned address.
@@ -89,11 +85,8 @@ class BanManager:
 	# @param value total number
 	
 	def setBanTotal(self, value):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			self.__banTotal = value
-		finally:
-			self.__lock.release()
 	
 	##
 	# Get the total number of banned address.
@@ -101,11 +94,8 @@ class BanManager:
 	# @return the total number
 	
 	def getBanTotal(self):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			return self.__banTotal
-		finally:
-			self.__lock.release()
 
 	##
 	# Returns a copy of the IP list.
@@ -113,11 +103,8 @@ class BanManager:
 	# @return IP list
 	
 	def getBanList(self):
-		try:
-			self.__lock.acquire()
-			return [m.getIP() for m in self.__banList]
-		finally:
-			self.__lock.release()
+		with self.__lock:
+			return self.__banList.keys()
 
 	##
 	# Returns normalized value
@@ -149,7 +136,7 @@ class BanManager:
 			return return_dict
 		self.__lock.acquire()
 		try:
-			for banData in self.__banList:
+			for banData in self.__banList.values():
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
 				question = ip.getPTR(
@@ -261,29 +248,31 @@ class BanManager:
 	# @return True if the IP address is not in the ban list
 	
 	def addBanTicket(self, ticket):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			# check already banned
-			for oldticket in self.__banList:
-				if ticket.getIP() == oldticket.getIP():
-					# if already permanent
-					btold, told = oldticket.getBanTime(self.__banTime), oldticket.getTime()
-					if btold == -1:
-						return False
-					# if given time is less than already banned time
-					btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
-					if btnew != -1 and tnew + btnew <= told + btold:
-						return False
-					# we have longest ban - set new (increment) ban time
-					oldticket.setTime(tnew)
-					oldticket.setBanTime(btnew)
+			fid = ticket.getID()
+			oldticket = self.__banList.get(fid)
+			if oldticket:
+				# if already permanent
+				btold, told = oldticket.getBanTime(self.__banTime), oldticket.getTime()
+				if btold == -1:
 					return False
+				# if given time is less than already banned time
+				btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
+				if btnew != -1 and tnew + btnew <= told + btold:
+					return False
+				# we have longest ban - set new (increment) ban time
+				oldticket.setTime(tnew)
+				oldticket.setBanTime(btnew)
+				return False
 			# not yet banned - add new
-			self.__banList.append(ticket)
+			self.__banList[fid] = ticket
 			self.__banTotal += 1
+			# correct next unban time:
+			eob = ticket.getEndOfBanTime(self.__banTime)
+			if self.__nextUnbanTime > eob:
+				self.__nextUnbanTime = eob
 			return True
-		finally:
-			self.__lock.release()
 
 	##
 	# Get the size of the ban list.
@@ -291,11 +280,7 @@ class BanManager:
 	# @return the size
 
 	def size(self):
-		try:
-			self.__lock.acquire()
-			return len(self.__banList)
-		finally:
-			self.__lock.release()
+		return len(self.__banList)
 
 	##
 	# Check if a ticket is in the list.
@@ -306,10 +291,7 @@ class BanManager:
 	# @return True if a ticket already exists
 	
 	def _inBanList(self, ticket):
-		for i in self.__banList:
-			if ticket.getIP() == i.getIP():
-				return True
-		return False
+		return ticket.getID() in self.__banList
 	
 	##
 	# Get the list of IP address to unban.
@@ -319,22 +301,39 @@ class BanManager:
 	# @return the list of ticket to unban
 	
 	def unBanList(self, time):
-		try:
-			self.__lock.acquire()
+		with self.__lock:
 			# Permanent banning
 			if self.__banTime < 0:
 				return list()
 
-			# Gets the list of ticket to remove.
-			unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)]
-			
+			# Check next unban time:
+			if self.__nextUnbanTime > time:
+				return list()
+
+			# Gets the list of ticket to remove (thereby correct next unban time).
+			unBanList = {}
+			self.__nextUnbanTime = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
+			for fid,ticket in self.__banList.iteritems():
+				# current time greater as end of ban - timed out:
+				eob = ticket.getEndOfBanTime(self.__banTime)
+				if time > eob:
+					unBanList[fid] = ticket
+				elif self.__nextUnbanTime > eob:
+					self.__nextUnbanTime = eob
+
 			# Removes tickets.
-			self.__banList = [ticket for ticket in self.__banList
-							  if ticket not in unBanList]
+			if len(unBanList):
+				if len(unBanList) / 2.0 <= len(self.__banList) / 3.0:
+					# few as 2/3 should be removed - remove particular items:
+					for fid in unBanList.iterkeys():
+						del self.__banList[fid]
+				else:
+					# create new dictionary without items to be deleted:
+					self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.iteritems() \
+						if fid not in unBanList)
 						
-			return unBanList
-		finally:
-			self.__lock.release()
+			# return list of tickets:
+			return unBanList.values()
 
 	##
 	# Flush the ban list.
@@ -343,28 +342,21 @@ class BanManager:
 	# @return the complete ban list
 	
 	def flushBanList(self):
-		try:
-			self.__lock.acquire()
-			uBList = self.__banList
-			self.__banList = list()
+		with self.__lock:
+			uBList = self.__banList.values()
+			self.__banList = dict()
 			return uBList
-		finally:
-			self.__lock.release()
 
 	##
-	# Gets the ticket for the specified IP.
+	# Gets the ticket for the specified ID (most of the time it is IP-address).
 	#
-	# @return the ticket for the IP or False.
-	def getTicketByIP(self, ip):
-		try:
-			self.__lock.acquire()
-
-			# Find the ticket the IP goes with and return it
-			for i, ticket in enumerate(self.__banList):
-				if ticket.getIP() == ip:
-					# Return the ticket after removing (popping)
-					# if from the ban list.
-					return self.__banList.pop(i)
-		finally:
-			self.__lock.release()
+	# @return the ticket or False.
+	def getTicketByID(self, fid):
+		with self.__lock:
+			try:
+				# Return the ticket after removing (popping)
+				# if from the ban list.
+				return self.__banList.pop(fid)
+			except KeyError:
+				pass
 		return None						  # if none found
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 8d07fcf3..4dc6ab88 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -96,8 +96,8 @@ class Ticket:
 	def setBanTime(self, value):
 		self._banTime = value;
 
-	def getBanTime(self, defaultBT = None):
-		return (self._banTime if not self._banTime is None else defaultBT);
+	def getBanTime(self, defaultBT=None):
+		return (self._banTime if self._banTime is not None else defaultBT)
 
 	def setBanCount(self, value):
 		self._banCount = value;
@@ -108,8 +108,16 @@ class Ticket:
 	def getBanCount(self):
 		return self._banCount;
 
-	def isTimedOut(self, time, defaultBT = None):
-		bantime = (self._banTime if not self._banTime is None else defaultBT);
+	def getEndOfBanTime(self, defaultBT=None):
+		bantime = (self._banTime if self._banTime is not None else defaultBT)
+		# permanent
+		if bantime == -1:
+			return 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
+		# unban time (end of ban):
+		return self._time + bantime
+
+	def isTimedOut(self, time, defaultBT=None):
+		bantime = (self._banTime if self._banTime is not None else defaultBT)
 		# permanent
 		if bantime == -1:
 			return False
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index f47dc848..c9f41c6c 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -70,7 +70,7 @@ class AddFailure(unittest.TestCase):
 			self.assertFalse(self.__banManager.addBanTicket(ticket2))
 			self.assertEqual(self.__banManager.size(), 1)
 			# pop ticket and check it was prolonged :
-			banticket = self.__banManager.getTicketByIP(ticket2.getIP())
+			banticket = self.__banManager.getTicketByID(ticket2.getID())
 			self.assertEqual(banticket.getTime(), ticket2.getTime())
 			self.assertEqual(banticket.getTime(), ticket2.getTime())
 			self.assertEqual(banticket.getBanTime(), ticket2.getBanTime(self.__banManager.getBanTime()))

From 8cba537f6ce75684d9d3fc7c33066f4876b78898 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 8 Sep 2016 18:29:45 +0200
Subject: [PATCH 174/623] code review and use new logger-signals for waiting; +
 regenerated man-files

---
 fail2ban/protocol.py                     |  2 +-
 fail2ban/tests/fail2banclienttestcase.py | 44 ++++++++------
 man/fail2ban-client.1                    | 77 ++++++++++++++----------
 man/fail2ban-regex.1                     | 18 +++---
 man/fail2ban-server.1                    |  8 +--
 man/fail2ban-testcases.1                 | 14 +++--
 6 files changed, 95 insertions(+), 68 deletions(-)

diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 6299794e..879183e5 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -50,7 +50,7 @@ protocol = [
 ["start", "starts the server and the jails"], 
 ["restart", "restarts the server"], 
 ["restart [--unban] [--if-exists] <JAIL>", "restarts the jail <JAIL> (alias for 'reload --restart ... <JAIL>')"], 
-["reload [--restart] [--unban] [--all]", "reloads the configuration without restarting of the server, the option '--restart' activates completely restarting of affected jails, thereby unbans IP addresses (if option '--unban' specified)"],
+["reload [--restart] [--unban] [--all]", "reloads the configuration without restarting of the server, the option '--restart' activates completely restarting of affected jails, thereby can unban IP addresses (if option '--unban' specified)"],
 ["reload [--restart] [--unban] [--if-exists] <JAIL>", "reloads the jail <JAIL>, or restarts it (if option '--restart' specified)"],
 ["stop", "stops all jails and terminate the server"], 
 ["unban --all", "unbans all IP addresses (in all jails and database)"],
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 1ce35d92..394a000f 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -712,8 +712,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MID_WAITTIME))
+			Utils.wait_for(lambda: \
+				self._is_logged("Reload finished.") and
+				self._is_logged("1 ticket(s) in 'test-jail1")
+				, MID_WAITTIME))
 		self.assertLogged("Added logfile: %r" % test1log)
+		self.assertLogged("[test-jail1] Ban 192.0.2.1")
 		
 		# enable both jails, 3 logs for jail1, etc...
 		# truncate test-log - we should not find unban/ban again by reload:
@@ -751,13 +755,16 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# test all will be found in jail1 and one in jail2:
 		self.assertTrue(
 			Utils.wait_for(lambda: \
-				self._is_logged("[test-jail1] Ban 192.0.2.2") and 
-				self._is_logged("[test-jail1] Ban 192.0.2.3") and 
-				self._is_logged("[test-jail1] Ban 192.0.2.4") and 
-				self._is_logged("[test-jail1] Ban 192.0.2.8") and 
-				self._is_logged("[test-jail2] Ban 192.0.2.4") and
-				self._is_logged("[test-jail2] Ban 192.0.2.8")
+				self._is_logged("2 ticket(s) in 'test-jail2") and
+				self._is_logged("5 ticket(s) in 'test-jail1")
 				, MID_WAITTIME))
+		self.assertLogged(
+			"[test-jail1] Ban 192.0.2.2",
+			"[test-jail1] Ban 192.0.2.3",
+			"[test-jail1] Ban 192.0.2.4",
+			"[test-jail1] Ban 192.0.2.8",
+			"[test-jail2] Ban 192.0.2.4",
+			"[test-jail2] Ban 192.0.2.8", all=True)
 		# test ips at all not visible for jail2:
 		self.assertNotLogged(
 			"[test-jail2] Found 192.0.2.2", 
@@ -775,9 +782,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"restart", "test-jail2")
 		self.assertTrue(
 			Utils.wait_for(lambda: \
-				self._is_logged("Jail 'test-jail2' started") and
-				self._is_logged("[test-jail2] Restore Ban 192.0.2.4") and
-				self._is_logged("[test-jail2] Restore Ban 192.0.2.8")
+				self._is_logged("Reload finished.") and
+				self._is_logged("Restore Ban") and
+				self._is_logged("2 ticket(s) in 'test-jail2")
 				, MID_WAITTIME))
 		# stop/start and unban/restore ban:
 		self.assertLogged(
@@ -794,8 +801,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.execSuccess(startparams,
 			"restart", "--unban", "test-jail2")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Jail 'test-jail2' started"),
-				MID_WAITTIME))
+			Utils.wait_for(lambda: \
+				self._is_logged("Reload finished.") and
+				self._is_logged("Jail 'test-jail2' started")
+				, MID_WAITTIME))
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
@@ -849,16 +858,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		))
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
-		# test "failure" regexp still available:
 		self.assertTrue(
 			Utils.wait_for(lambda: \
-				self._is_logged("[test-jail1] 192.0.2.1 already banned") and
-				self._is_logged("[test-jail1] Ban 192.0.2.6")
+				self._is_logged("6 ticket(s) in 'test-jail1") and
+				self._is_logged("[test-jail1] 192.0.2.1 already banned")
 				, MID_WAITTIME))
+		# test "failure" regexp still available:
 		self.assertLogged(
 			"[test-jail1] Found 192.0.2.1",
-			"[test-jail1] Found 192.0.2.6", all=True
-		)
+			"[test-jail1] Found 192.0.2.6",
+			"[test-jail1] 192.0.2.1 already banned",
+			"[test-jail1] Ban 192.0.2.6", all=True)
 		# test "error" regexp no more available:
 		self.assertNotLogged("[test-jail1] Found 192.0.2.5")
 
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 0da5fb02..33bce652 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
-.TH FAIL2BAN-CLIENT "1" "July 2016" "fail2ban-client v0.10.0a1" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
+.TH FAIL2BAN-CLIENT "1" "September 2016" "fail2ban-client v0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
-[\fIOPTIONS\fR] \fI<COMMAND>\fR
+[\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.10.0a1 reads log file that contains password failure report
+Fail2Ban v0.10.0a2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
@@ -68,17 +68,36 @@ starts the server and the jails
 \fBrestart\fR
 restarts the server
 .TP
-\fBreload\fR
-reloads the configuration without
-restart
+\fBrestart [\-\-unban] [\-\-if\-exists] <JAIL>\fR
+restarts the jail <JAIL> (alias
+for 'reload \fB\-\-restart\fR ... <JAIL>')
 .TP
-\fBreload <JAIL>\fR
-reloads the jail <JAIL>
+\fBreload [\-\-restart] [\-\-unban] [\-\-all]\fR
+reloads the configuration without
+restarting of the server, the
+option '\-\-restart' activates
+completely restarting of affected
+jails, thereby can unban IP
+addresses (if option '\-\-unban'
+specified)
+.TP
+\fBreload [\-\-restart] [\-\-unban] [\-\-if\-exists] <JAIL>\fR
+reloads the jail <JAIL>, or
+restarts it (if option '\-\-restart'
+specified)
 .TP
 \fBstop\fR
 stops all jails and terminate the
 server
 .TP
+\fBunban \fB\-\-all\fR\fR
+unbans all IP addresses (in all
+jails and database)
+.TP
+\fBunban <IP> ... <IP>\fR
+unbans <IP> (in all jails and
+database)
+.TP
 \fBstatus\fR
 gets the current status of the
 server
@@ -248,9 +267,8 @@ for <JAIL>
 \fBset <JAIL> maxlines <LINES>\fR
 sets the number of <LINES> to
 buffer for regex search for <JAIL>
-.IP
-set <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]
-.IP
+.TP
+\fBset <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]\fR
 adds a new action named <ACT> for
 <JAIL>. Optionally for a Python
 based action, a <PYTHONFILE> and
@@ -262,45 +280,38 @@ removes the action <ACT> from
 <JAIL>
 .IP
 COMMAND ACTION CONFIGURATION
-.IP
-set <JAIL> action <ACT> actionstart <CMD>
-.IP
+.TP
+\fBset <JAIL> action <ACT> actionstart <CMD>\fR
 sets the start command <CMD> of
 the action <ACT> for <JAIL>
-.IP
-set <JAIL> action <ACT> actionstop <CMD> sets the stop command <CMD> of the
-.IP
+.TP
+\fBset <JAIL> action <ACT> actionstop <CMD> sets the stop command <CMD> of the\fR
 action <ACT> for <JAIL>
-.IP
-set <JAIL> action <ACT> actioncheck <CMD>
-.IP
+.TP
+\fBset <JAIL> action <ACT> actioncheck <CMD>\fR
 sets the check command <CMD> of
 the action <ACT> for <JAIL>
 .TP
 \fBset <JAIL> action <ACT> actionban <CMD>\fR
 sets the ban command <CMD> of the
 action <ACT> for <JAIL>
-.IP
-set <JAIL> action <ACT> actionunban <CMD>
-.IP
+.TP
+\fBset <JAIL> action <ACT> actionunban <CMD>\fR
 sets the unban command <CMD> of
 the action <ACT> for <JAIL>
-.IP
-set <JAIL> action <ACT> timeout <TIMEOUT>
-.IP
+.TP
+\fBset <JAIL> action <ACT> timeout <TIMEOUT>\fR
 sets <TIMEOUT> as the command
 timeout in seconds for the action
 <ACT> for <JAIL>
 .IP
 GENERAL ACTION CONFIGURATION
-.IP
-set <JAIL> action <ACT> <PROPERTY> <VALUE>
-.IP
+.TP
+\fBset <JAIL> action <ACT> <PROPERTY> <VALUE>\fR
 sets the <VALUE> of <PROPERTY> for
 the action <ACT> for <JAIL>
-.IP
-set <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]
-.IP
+.TP
+\fBset <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]\fR
 calls the <METHOD> with
 <JSONKWARGS> for the action <ACT>
 for <JAIL>
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index f954cb3a..342d18f7 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
-.TH FAIL2BAN-REGEX "1" "July 2016" "fail2ban-regex 0.10.0a1" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
+.TH FAIL2BAN-REGEX "1" "September 2016" "fail2ban-regex 0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
 .B fail2ban-regex
-[\fIOPTIONS\fR] \fI<LOG> <REGEX> \fR[\fIIGNOREREGEX\fR]
+[\fI\,OPTIONS\/\fR] \fI\,<LOG> <REGEX> \/\fR[\fI\,IGNOREREGEX\/\fR]
 .SH DESCRIPTION
 Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
@@ -16,7 +16,7 @@ string
 a string representing a log line
 .TP
 filename
-path to a log file (\fI/var/log/auth.log\fP)
+path to a log file (\fI\,/var/log/auth.log\/\fP)
 .TP
 "systemd\-journal"
 search systemd journal (systemd\-python required)
@@ -42,23 +42,23 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fIDATEPATTERN\fR
+\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR
 set custom pattern used to match date/times
 .TP
-\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fIENCODING\fR
+\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fI\,ENCODING\/\fR
 File encoding. Default: system locale
 .TP
 \fB\-r\fR, \fB\-\-raw\fR
 Raw hosts, don't resolve dns
 .TP
-\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fIMAXLINES\fR
+\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fI\,MAXLINES\/\fR
 maxlines for multi\-line regex
 .TP
-\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fIJOURNALMATCH\fR
+\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fI\,JOURNALMATCH\/\fR
 journalctl style matches overriding filter file.
 "systemd\-journal" only
 .TP
-\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fILOG_LEVEL\fR
+\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR
 Log level for the Fail2Ban logger to use
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 3b970024..de8ba6a4 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
-.TH FAIL2BAN-SERVER "1" "July 2016" "fail2ban-server v0.10.0a1" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
+.TH FAIL2BAN-SERVER "1" "September 2016" "fail2ban-server v0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
-[\fIOPTIONS\fR]
+[\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.10.0a1 reads log file that contains password failure report
+Fail2Ban v0.10.0a2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index 9089d1ed..26dbae96 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,10 +1,10 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.44.1.
-.TH FAIL2BAN-TESTCASES "1" "July 2016" "fail2ban-testcases 0.10.0a1" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
+.TH FAIL2BAN-TESTCASES "1" "September 2016" "fail2ban-testcases 0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS
 .B fail2ban-testcases
-[\fIOPTIONS\fR] [\fIregexps\fR]
+[\fI\,OPTIONS\/\fR] [\fI\,regexps\/\fR]
 .SH DESCRIPTION
 Script to run Fail2Ban tests battery
 .SH OPTIONS
@@ -15,9 +15,15 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fILOG_LEVEL\fR
+\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR
 Log level for the logger to use during running tests
 .TP
+\fB\-v\fR VERBOSITY, \fB\-\-verbosity\fR=\fI\,VERBOSITY\/\fR
+Set numerical level of verbosity (0..4)
+.TP
+\fB\-\-log\-direct\fR
+Prevent lazy logging inside tests
+.TP
 \fB\-n\fR, \fB\-\-no\-network\fR
 Do not run tests that require the network
 .TP

From d1ef33cc45e411b4bdf44b1a048dd5e5968c2661 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 8 Sep 2016 19:45:47 +0200
Subject: [PATCH 175/623] New command action parameter `actionrepair` - command
 executed in order to restore sane environment in error case of `actioncheck`.
 # [WARNING] TODO: be sure all banactions get a repair command, because
 otherwise stop/start will theoretically remove all the bans, but the tickets
 are still in BanManager, so in case of new failures it will not be banned,
 because "already banned" will happen.

---
 fail2ban/client/actionreader.py  |  1 +
 fail2ban/server/action.py        | 44 ++++++++++++++++++++++++--------
 fail2ban/tests/actiontestcase.py | 23 +++++++++++++++++
 3 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index 698360ac..2e02c4aa 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -39,6 +39,7 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionstart": ["string", None],
 		"actionstop": ["string", None],
 		"actioncheck": ["string", None],
+		"actionrepair": ["string", None],
 		"actionban": ["string", None],
 		"actionunban": ["string", None],
 	}
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 702ad397..9f539e2c 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -200,6 +200,8 @@ class CommandAction(ActionBase):
 	Attributes
 	----------
 	actionban
+	actioncheck
+	actionrepair
 	actionstart
 	actionstop
 	actionunban
@@ -217,6 +219,8 @@ class CommandAction(ActionBase):
 	actionunban = ''
 	## Command executed in order to check requirements.
 	actioncheck = ''
+	## Command executed in order to restore sane environment in error case.
+	actionrepair = ''
 	## Command executed in order to stop the system.
 	actionstop = ''
 
@@ -271,16 +275,18 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		# check valid tags in properties (raises ValueError if self recursion, etc.):
+		res = True
 		try:
 			# common (resp. ipv4):
 			startCmd = self.replaceTag('<actionstart>', self._properties, 
 				conditional='family=inet4', cache=self.__substCache)
-			res = self.executeCmd(startCmd, self.timeout)
+			if startCmd:
+				res &= self.executeCmd(startCmd, self.timeout)
 			# start ipv6 actions if available:
 			if allowed_ipv6:
 				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
 					conditional='family=inet6', cache=self.__substCache)
-				if startCmd6 != startCmd:
+				if startCmd6 and startCmd6 != startCmd:
 					res &= self.executeCmd(startCmd6, self.timeout)
 			if not res:
 				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
@@ -323,15 +329,17 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
+		res = True
 		# common (resp. ipv4):
 		stopCmd = self.replaceTag('<actionstop>', self._properties, 
 			conditional='family=inet4', cache=self.__substCache)
-		res = self.executeCmd(stopCmd, self.timeout)
+		if stopCmd:
+			res &= self.executeCmd(stopCmd, self.timeout)
 		# ipv6 actions if available:
 		if allowed_ipv6:
 			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
 				conditional='family=inet6', cache=self.__substCache)
-			if stopCmd6 != stopCmd:
+			if stopCmd6 and stopCmd6 != stopCmd:
 				res &= self.executeCmd(stopCmd6, self.timeout)
 		if not res:
 			raise RuntimeError("Error stopping action")
@@ -520,14 +528,28 @@ class CommandAction(ActionBase):
 
 		checkCmd = self.replaceTag('<actioncheck>', self._properties, 
 			conditional=conditional, cache=self.__substCache)
-		if not self.executeCmd(checkCmd, self.timeout):
-			self._logSys.error(
-				"Invariant check failed. Trying to restore a sane environment")
-			self.stop()
-			self.start()
+		if checkCmd:
 			if not self.executeCmd(checkCmd, self.timeout):
-				self._logSys.critical("Unable to restore environment")
-				return False
+				self._logSys.error(
+					"Invariant check failed. Trying to restore a sane environment")
+				# try to find repair command, if exists - exec it:
+				repairCmd = self.replaceTag('<actionrepair>', self._properties, 
+					conditional=conditional, cache=self.__substCache)
+				if repairCmd:
+					if not self.executeCmd(repairCmd, self.timeout):
+						self._logSys.critical("Unable to restore environment")
+						return False
+				else:
+					# no repair command, try to restart action...
+					# [WARNING] TODO: be sure all banactions get a repair command, because
+					#    otherwise stop/start will theoretically remove all the bans,
+					#    but the tickets are still in BanManager, so in case of new failures
+					#    it will not be banned, because "already banned" will happen.
+					self.stop()
+					self.start()
+				if not self.executeCmd(checkCmd, self.timeout):
+					self._logSys.critical("Unable to restore environment")
+					return False
 
 		# Replace static fields
 		realCmd = self.replaceTag(cmd, self._properties, 
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 57c4856a..166450bb 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -277,6 +277,24 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertRaises(RuntimeError, self.__action.ban, {'ip': None})
 		self.assertLogged('Unable to restore environment')
 
+	def testExecuteActionCheckRepairEnvironment(self):
+		self.__action.actionstart = ""
+		self.__action.actionstop = ""
+		self.__action.actionban = "rm /tmp/fail2ban.test"
+		self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]"
+		self.__action.actionrepair = "echo 'repair ...'; touch /tmp/fail2ban.test"
+		# 1st time with success repair:
+		self.__action.ban({'ip': None})
+		self.assertLogged("Invariant check failed. Trying", "echo 'repair ...'", all=True)
+		self.pruneLog()
+		# 2nd time failed (not really repaired):
+		self.__action.actionrepair = "echo 'repair ...'"
+		self.assertRaises(RuntimeError, self.__action.ban, {'ip': None})
+		self.assertLogged(
+			"Invariant check failed. Trying", 
+			"echo 'repair ...'", 
+			"Unable to restore environment", all=True)
+
 	def testExecuteActionChangeCtags(self):
 		self.assertRaises(AttributeError, getattr, self.__action, "ROST")
 		self.__action.ROST = "192.0.2.0"
@@ -294,7 +312,12 @@ class CommandActionTest(LogCaptureTestCase):
 	def testExecuteActionStartEmpty(self):
 		self.__action.actionstart = ""
 		self.__action.start()
+		self.assertTrue(self.__action.executeCmd(""))
 		self.assertLogged('Nothing to do')
+		self.pruneLog()
+		self.assertTrue(self.__action._processCmd(""))
+		self.assertLogged('Nothing to do')
+		self.pruneLog()
 
 	def testExecuteIncorrectCmd(self):
 		CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null')

From 4fb511294e01d33e934c32b7d458088e19d42343 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 8 Sep 2016 23:56:32 +0200
Subject: [PATCH 176/623] temp commit: reload now supported actions and action
 reloading (parameters, unban obsolete removed actions, etc.)

---
 fail2ban/client/actionreader.py          |  1 +
 fail2ban/server/action.py                | 88 +++++++++++++---------
 fail2ban/server/actions.py               | 96 ++++++++++++++++++------
 fail2ban/server/banmanager.py            |  9 +++
 fail2ban/server/database.py              | 12 ++-
 fail2ban/server/filter.py                |  3 -
 fail2ban/server/server.py                |  6 +-
 fail2ban/tests/fail2banclienttestcase.py | 27 ++++++-
 8 files changed, 171 insertions(+), 71 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index 2e02c4aa..e5025fa3 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -38,6 +38,7 @@ class ActionReader(DefinitionInitConfigReader):
 	_configOpts = {
 		"actionstart": ["string", None],
 		"actionstop": ["string", None],
+		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
 		"actionban": ["string", None],
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 9f539e2c..976adb20 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -201,6 +201,7 @@ class CommandAction(ActionBase):
 	----------
 	actionban
 	actioncheck
+	actionreload
 	actionrepair
 	actionstart
 	actionstop
@@ -210,24 +211,35 @@ class CommandAction(ActionBase):
 
 	_escapedTags = set(('matches', 'ipmatches', 'ipjailmatches'))
 
-	timeout = 60
-	## Command executed in order to initialize the system.
-	actionstart = ''
-	## Command executed when an IP address gets banned.
-	actionban = ''
-	## Command executed when an IP address gets removed.
-	actionunban = ''
-	## Command executed in order to check requirements.
-	actioncheck = ''
-	## Command executed in order to restore sane environment in error case.
-	actionrepair = ''
-	## Command executed in order to stop the system.
-	actionstop = ''
+	def clearAllParams(self):
+		""" Clear all lists/dicts parameters (used by reloading)
+		"""
+		self.__init = 1
+		try:
+			self.timeout = 60
+			## Command executed in order to initialize the system.
+			self.actionstart = ''
+			## Command executed when an IP address gets banned.
+			self.actionban = ''
+			## Command executed when an IP address gets removed.
+			self.actionunban = ''
+			## Command executed in order to check requirements.
+			self.actioncheck = ''
+			## Command executed in order to restore sane environment in error case.
+			self.actionrepair = ''
+			## Command executed in order to stop the system.
+			self.actionstop = ''
+			## Command executed in case of reloading action.
+			self.actionreload = ''
+		finally:
+			self.__init = 0
 
 	def __init__(self, jail, name):
 		super(CommandAction, self).__init__(jail, name)
+		self.__init = 1
 		self.__properties = None
 		self.__substCache = {}
+		self.clearAllParams()
 		self._logSys.debug("Created %s" % self.__class__)
 
 	@classmethod
@@ -235,7 +247,7 @@ class CommandAction(ActionBase):
 		return NotImplemented # Standard checks
 
 	def __setattr__(self, name, value):
-		if not name.startswith('_') and not callable(value):
+		if not name.startswith('_') and not self.__init and not callable(value):
 			# special case for some pasrameters:
 			if name in ('timeout', 'bantime'):
 				value = str(MyTime.str2seconds(value))
@@ -268,8 +280,8 @@ class CommandAction(ActionBase):
 	def _substCache(self):
 		return self.__substCache
 
-	def start(self):
-		"""Executes the "actionstart" command.
+	def _executeOperation(self, tag, operation):
+		"""Executes the operation commands (like "actionstart", "actionstop", etc).
 
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
@@ -278,20 +290,28 @@ class CommandAction(ActionBase):
 		res = True
 		try:
 			# common (resp. ipv4):
-			startCmd = self.replaceTag('<actionstart>', self._properties, 
+			startCmd = self.replaceTag(tag, self._properties, 
 				conditional='family=inet4', cache=self.__substCache)
 			if startCmd:
 				res &= self.executeCmd(startCmd, self.timeout)
 			# start ipv6 actions if available:
 			if allowed_ipv6:
-				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
+				startCmd6 = self.replaceTag(tag, self._properties, 
 					conditional='family=inet6', cache=self.__substCache)
 				if startCmd6 and startCmd6 != startCmd:
 					res &= self.executeCmd(startCmd6, self.timeout)
 			if not res:
-				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
+				raise RuntimeError("Error %s action %s/%s" % (operation, self._jail, self._name,))
 		except ValueError as e:
-			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
+			raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e))
+
+	def start(self):
+		"""Executes the "actionstart" command.
+
+		Replace the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		return self._executeOperation('<actionstart>', 'starting')
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -329,20 +349,20 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		res = True
-		# common (resp. ipv4):
-		stopCmd = self.replaceTag('<actionstop>', self._properties, 
-			conditional='family=inet4', cache=self.__substCache)
-		if stopCmd:
-			res &= self.executeCmd(stopCmd, self.timeout)
-		# ipv6 actions if available:
-		if allowed_ipv6:
-			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
-				conditional='family=inet6', cache=self.__substCache)
-			if stopCmd6 and stopCmd6 != stopCmd:
-				res &= self.executeCmd(stopCmd6, self.timeout)
-		if not res:
-			raise RuntimeError("Error stopping action")
+		return self._executeOperation('<actionstop>', 'stopping')
+
+	def reload(self, **kwargs):
+		"""Executes the "actionreload" command.
+
+		Parameters
+		----------
+		kwargs : dict
+		  Currently unused, because CommandAction do not support initOpts
+
+		Replaces the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
 	def substituteRecursiveTags(cls, inptags, conditional=''):
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 244d1e68..09b6823d 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -36,7 +36,7 @@ from collections import Mapping
 try:
 	from collections import OrderedDict
 except ImportError:
-	OrderedDict = None
+	OrderedDict = dict
 
 from .banmanager import BanManager
 from .jailthread import JailThread
@@ -81,14 +81,11 @@ class Actions(JailThread, Mapping):
 		JailThread.__init__(self)
 		## The jail which contains this action.
 		self._jail = jail
-		if OrderedDict is not None:
-			self._actions = OrderedDict()
-		else:
-			self._actions = dict()
+		self._actions = OrderedDict()
 		## The ban manager.
 		self.__banManager = BanManager()
 
-	def add(self, name, pythonModule=None, initOpts=None):
+	def add(self, name, pythonModule=None, initOpts=None, reload=False):
 		"""Adds a new action.
 
 		Add a new action if not already present, defaulting to standard
@@ -116,7 +113,17 @@ class Actions(JailThread, Mapping):
 		"""
 		# Check is action name already exists
 		if name in self._actions:
-			raise ValueError("Action %s already exists" % name)
+			if not reload:
+				raise ValueError("Action %s already exists" % name)
+			# don't create new action if reload supported:
+			action = self._actions[name]
+			if hasattr(action, 'reload'):
+				# don't execute reload right now, reload after all parameters are actualized
+				if hasattr(action, 'clearAllParams'):
+					action.clearAllParams()
+					self._reload_actions[name] = initOpts
+				return
+		## Create new action:
 		if pythonModule is None:
 			action = CommandAction(self._jail, name)
 		else:
@@ -138,6 +145,27 @@ class Actions(JailThread, Mapping):
 			action = customActionModule.Action(self._jail, name, **initOpts)
 		self._actions[name] = action
 
+	def reload(self, begin=True):
+		""" Begin or end of reloading resp. refreshing of all parameters
+		"""
+		if begin:
+			self._reload_actions = dict()
+		else:
+			if hasattr(self, '_reload_actions'):
+				# reload actions after all parameters set via stream:
+				for name, initOpts in self._reload_actions.iteritems():
+					if name in self._actions:
+						self._actions[name].reload(**initOpts if initOpts else {})
+				# remove obsolete actions (untouched by reload process):
+				delacts = OrderedDict((name, action) for name, action in self._actions.iteritems()
+					if name not in self._reload_actions)
+				if len(delacts):
+					# unban all tickets using remove action only:
+					self.__flushBan(db=False, actions=delacts)
+					# stop and remove it:
+					self.stopActions(actions=delacts)
+				delattr(self, '_reload_actions')
+
 	def __getitem__(self, name):
 		try:
 			return self._actions[name]
@@ -215,6 +243,24 @@ class Actions(JailThread, Mapping):
 		return 1
 
 
+	def stopActions(self, actions=None):
+		"""Stops the actions in reverse sequence (optionally filtered)
+		"""
+		if actions is None:
+			actions = self._actions
+		revactions = actions.items()
+		revactions.reverse()
+		for name, action in revactions:
+			try:
+				action.stop()
+			except Exception as e:
+				logSys.error("Failed to stop jail '%s' action '%s': %s",
+					self._jail.name, name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			del self._actions[name]
+			logSys.debug("%s: action %s terminated", self._jail.name, name)
+
+
 	def run(self):
 		"""Main loop for Threading.
 
@@ -239,18 +285,9 @@ class Actions(JailThread, Mapping):
 				continue
 			if not Utils.wait_for(self.__checkBan, self.sleeptime):
 				self.__checkUnBan()
+		
 		self.__flushBan()
-
-		actions = self._actions.items()
-		actions.reverse()
-		for name, action in actions:
-			try:
-				action.stop()
-			except Exception as e:
-				logSys.error("Failed to stop jail '%s' action '%s': %s",
-					self._jail.name, name, e,
-					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-		logSys.debug(self._jail.name + ": action terminated")
+		self.stopActions()
 		return True
 
 	def __getBansMerged(self, mi, overalljails=False):
@@ -355,26 +392,33 @@ class Actions(JailThread, Mapping):
 				cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __flushBan(self, db=False):
+	def __flushBan(self, db=False, actions=None):
 		"""Flush the ban list.
 
 		Unban all IP address which are still in the banning list.
+
+		If actions specified, don't flush list - just execute unban for 
+		given actions (reload, obsolete resp. removed actions).
 		"""
-		logSys.debug("Flush ban list")
-		lst = self.__banManager.flushBanList()
+		if actions is None:
+			logSys.debug("Flush ban list")
+			lst = self.__banManager.flushBanList()
+		else:
+			lst = iter(self.__banManager)
+		cnt = 0
 		for ticket in lst:
 			# delete ip from database also:
 			if db and self._jail.database is not None:
 				ip = str(ticket.getIP())
 				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
-			self.__unBan(ticket)
-		cnt = len(lst)
+			self.__unBan(ticket, actions=actions)
+			cnt += 1
 		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
 			cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __unBan(self, ticket):
+	def __unBan(self, ticket, actions=None):
 		"""Unbans host corresponding to the ticket.
 
 		Executes the actions in order to unban the host given in the
@@ -385,13 +429,15 @@ class Actions(JailThread, Mapping):
 		ticket : FailTicket
 			Ticket of failures of which to unban
 		"""
+		if actions is None:
+			actions = self._actions
 		aInfo = dict()
 		aInfo["ip"] = ticket.getIP()
 		aInfo["failures"] = ticket.getAttempt()
 		aInfo["time"] = ticket.getTime()
 		aInfo["matches"] = "".join(ticket.getMatches())
 		logSys.notice("[%s] Unban %s" % (self._jail.name, aInfo["ip"]))
-		for name, action in self._actions.iteritems():
+		for name, action in actions.iteritems():
 			try:
 				action.unban(aInfo.copy())
 			except Exception as e:
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index e0a9e5ca..41e202ef 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -106,6 +106,15 @@ class BanManager:
 		with self.__lock:
 			return self.__banList.keys()
 
+	##
+	# Returns a iterator to ban list (used in reload, so idle).
+	#
+	# @return ban list iterator
+	
+	def __iter__(self):
+		with self.__lock:
+			return self.__banList.itervalues()
+
 	##
 	# Returns normalized value
 	#
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index fc544be4..53db84a9 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -604,16 +604,14 @@ class Fail2BanDb(object):
 		results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
 
 		if results:
-			matches = []
-			failures = 0
 			for banip, timeofban, data in results:
-				#TODO: Implement data parts once arbitrary match keys completed
-				ticket = FailTicket(banip, timeofban, matches)
-				ticket.setAttempt(failures)
 				matches = []
 				failures = 0
-				matches.extend(data['matches'])
-				failures += data['failures']
+				if isinstance(data['matches'], list):
+					matches.extend(data['matches'])
+				if data['failures']:
+					failures += data['failures']
+				ticket = FailTicket(banip, timeofban, matches)
 				ticket.setAttempt(failures)
 				tickets.append(ticket)
 
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 1c466c01..6b0599f5 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -117,14 +117,11 @@ class Filter(JailThread):
 				self._reload_logs = dict((k, 1) for k in self.getLogPaths())
 		else:
 			if hasattr(self, '_reload_logs'):
-				dellogs = dict()
 				# if it was not reloaded - remove obsolete log file:
 				for path in self._reload_logs:
 					self.delLogPath(path)
 				delattr(self, '_reload_logs')
 
-
-
 	##
 	# Add a regular expression which matches the failure.
 	#
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 041b0983..fc91ca73 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -267,6 +267,7 @@ class Server:
 						jail.idle = True
 						self.__reload_state[jn] = jail
 						jail.filter.reload(begin=True)
+						jail.actions.reload(begin=True)
 				pass
 		else:
 			# end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started:
@@ -280,6 +281,7 @@ class Server:
 					else:
 						# commit (reload was finished):
 						jail.filter.reload(begin=False)
+						jail.actions.reload(begin=False)
 				for jn in deljails:
 					self.delJail(jn)
 			self.__reload_state = {}
@@ -416,7 +418,9 @@ class Server:
 	
 	# Action
 	def addAction(self, name, value, *args):
-		self.__jails[name].actions.add(value, *args)
+		## create (or reload) jail action:
+		self.__jails[name].actions.add(value, *args, 
+			reload=name in self.__reload_state)
 	
 	def getActions(self, name):
 		return self.__jails[name].actions
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 394a000f..d818e252 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -677,6 +677,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		test2log = pjoin(tmp, "test2.log")
 		test3log = pjoin(tmp, "test3.log")
 
+		os.mkdir(pjoin(cfg, "action.d"))
+		def _write_action_cfg(actname="test-action1"):
+			fn = pjoin(cfg, "action.d", "%s.conf" % actname)
+			_write_file(fn, "w",
+				"[Definition]",
+				"actionstart = echo '[<name>] %s: ** start'" % actname,
+				"actionstop =  echo '[<name>] %s: -- unban <ip>'" % actname,
+				"actionreload = echo '[<name>] %s: ** reload'" % actname,
+				"actionban =   echo '[<name>] %s: ++ ban <ip>'" % actname,
+				"actionunban = echo '[<name>] %s: // stop'" % actname,
+			)
+			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+				_out_file(fn)
+
 		def _write_jail_cfg(enabled=[1, 2]):
 			_write_file(pjoin(cfg, "jail.conf"), "w",
 				"[INCLUDES]", "",
@@ -686,7 +700,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",
 				"",
-				"[test-jail1]", "backend = polling", "filter =", "action =",
+				"[test-jail1]", "backend = polling", "filter =", 
+				"action = test-action1[name='%(__name__)s']",
+				"         test-action2[name='%(__name__)s']",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
@@ -701,6 +717,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 				_out_file(pjoin(cfg, "jail.conf"))
 
+		# create default test actions:
+		_write_action_cfg(actname="test-action1")
+		_write_action_cfg(actname="test-action2")
+
 		_write_jail_cfg(enabled=[1])
 		_write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3))
 		_write_file(test2log, "w")
@@ -737,6 +757,11 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Added logfile: %r" % test2log, 
 			"Added logfile: %r" % test3log, all=True)
+		# test actions reloaded:
+		self.assertLogged(
+			"echo '[test-jail1] test-action1: ** reload' -- returned successfully", 
+			"echo '[test-jail1] test-action2: ** reload' -- returned successfully", all=True)
+
 		# test 1 new jail:
 		self.assertLogged(
 			"Creating new jail 'test-jail2'",

From 8c4eebc3e315dd6b7602d8fbebf900947cf3e8f2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 00:39:48 +0200
Subject: [PATCH 177/623] reload actions amend, code review and test cases
 extended for update/start/stop of actions by reloading

---
 fail2ban/server/actions.py               | 10 ++-
 fail2ban/server/asyncserver.py           |  5 +-
 fail2ban/server/database.py              | 16 ++---
 fail2ban/server/server.py                |  6 ++
 fail2ban/server/ticket.py                |  7 +-
 fail2ban/tests/fail2banclienttestcase.py | 83 ++++++++++++++++++------
 6 files changed, 91 insertions(+), 36 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 09b6823d..5ac7b9a9 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -430,15 +430,19 @@ class Actions(JailThread, Mapping):
 			Ticket of failures of which to unban
 		"""
 		if actions is None:
-			actions = self._actions
+			unbactions = self._actions
+		else:
+			unbactions = actions
 		aInfo = dict()
 		aInfo["ip"] = ticket.getIP()
 		aInfo["failures"] = ticket.getAttempt()
 		aInfo["time"] = ticket.getTime()
 		aInfo["matches"] = "".join(ticket.getMatches())
-		logSys.notice("[%s] Unban %s" % (self._jail.name, aInfo["ip"]))
-		for name, action in actions.iteritems():
+		if actions is None:
+			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
+		for name, action in unbactions.iteritems():
 			try:
+				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"])
 				action.unban(aInfo.copy())
 			except Exception as e:
 				logSys.error(
diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index bb237cd7..ffb20503 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -199,6 +199,7 @@ class AsyncServer(asyncore.dispatcher):
 
 
 	def close(self):
+		stopflg = False
 		if self.__active:
 			self.__loop = False
 			asyncore.dispatcher.close(self)
@@ -206,11 +207,13 @@ class AsyncServer(asyncore.dispatcher):
 			# for the server leaves loop, before remove socket
 			if threading.current_thread() != self.__worker:
 				Utils.wait_for(lambda: not self.__active, 1)
+			stopflg = True
 		# Remove socket (file) only if it was created:
 		if self.__init and os.path.exists(self.__sock):
 			self._remove_sock()
 			logSys.debug("Removed socket file " + self.__sock)
-		logSys.debug("Socket shutdown")
+		if stopflg:
+			logSys.debug("Socket shutdown")
 		self.__active = False
 
 	##
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 53db84a9..24445f3d 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -223,6 +223,11 @@ class Fail2BanDb(object):
 				cur.execute("PRAGMA journal_mode = MEMORY")
 			cur.close()
 
+	def close(self):
+		logSys.debug("Close connection to database ...")
+		self._db.close()
+		logSys.info("Connection to database closed.")
+
 	@property
 	def filename(self):
 		"""File name of SQLite3 database file.
@@ -605,14 +610,9 @@ class Fail2BanDb(object):
 
 		if results:
 			for banip, timeofban, data in results:
-				matches = []
-				failures = 0
-				if isinstance(data['matches'], list):
-					matches.extend(data['matches'])
-				if data['failures']:
-					failures += data['failures']
-				ticket = FailTicket(banip, timeofban, matches)
-				ticket.setAttempt(failures)
+				# logSys.debug('restore ticket   %r, %r, %r', banip, timeofban, data)
+				ticket = FailTicket(banip, timeofban, data=data)
+				# logSys.debug('restored ticket: %r', ticket)
 				tickets.append(ticket)
 
 		return tickets if ip is None else ticket
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index fc91ca73..97345331 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -171,6 +171,12 @@ class Server:
 		# Now stop all the jails
 		self.stopAllJail()
 
+		# Explicit close database (server can leave in a thread, 
+		# so delayed GC can prevent commiting changes)
+		if self.__db:
+			self.__db.close()
+			self.__db = None
+
 		# Only now shutdown the logging.
 		if self.__logTarget is not None:
 			with self.__loggingLock:
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 4dc6ab88..8ff6a780 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -51,13 +51,12 @@ class Ticket:
 		self._banCount = 0;
 		self._banTime = None;
 		self._time = time if time is not None else MyTime.time()
-		self._data = {'matches': [], 'failures': 0}
-		self._data.update(data)
+		self._data = {'matches': matches or [], 'failures': 0}
+		if data is not None:
+			self._data.update(data)
 		if ticket:
 			# ticket available - copy whole information from ticket:
 			self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__)
-		else:
-			self._data['matches'] = matches or []
 
 	def __str__(self):
 		return "%s: ip=%s time=%s #attempts=%d matches=%r" % \
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index d818e252..2804fcd4 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -678,20 +678,24 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		test3log = pjoin(tmp, "test3.log")
 
 		os.mkdir(pjoin(cfg, "action.d"))
-		def _write_action_cfg(actname="test-action1"):
+		def _write_action_cfg(actname="test-action1", allow=True, 
+			start="", reload="", ban="", unban="", stop=""):
 			fn = pjoin(cfg, "action.d", "%s.conf" % actname)
+			if not allow:
+				os.remove(fn)
+				return
 			_write_file(fn, "w",
 				"[Definition]",
-				"actionstart = echo '[<name>] %s: ** start'" % actname,
-				"actionstop =  echo '[<name>] %s: -- unban <ip>'" % actname,
-				"actionreload = echo '[<name>] %s: ** reload'" % actname,
-				"actionban =   echo '[<name>] %s: ++ ban <ip>'" % actname,
-				"actionunban = echo '[<name>] %s: // stop'" % actname,
+				"actionstart =  echo '[<name>] %s: ** start'" % actname, start,
+				"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
+				"actionban =    echo '[<name>] %s: ++ ban <ip>'" % actname, ban,
+				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
+				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
 			)
-			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			if DefLogSys.level <= logging.DEBUG:  # if DEBUG
 				_out_file(fn)
 
-		def _write_jail_cfg(enabled=[1, 2]):
+		def _write_jail_cfg(enabled=(1, 2), actions=()):
 			_write_file(pjoin(cfg, "jail.conf"), "w",
 				"[INCLUDES]", "",
 				"[DEFAULT]", "",
@@ -701,8 +705,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"failregex = ^\s*failure (401|403) from <HOST>",
 				"",
 				"[test-jail1]", "backend = polling", "filter =", 
-				"action = test-action1[name='%(__name__)s']",
-				"         test-action2[name='%(__name__)s']",
+				"action = ",
+				"         test-action1[name='%(__name__)s']" if 1 in actions else "",
+				"         test-action2[name='%(__name__)s']" if 2 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
@@ -710,24 +715,25 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"            ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
 				"enabled = true" if 1 in enabled else "",
 				"",
-				"[test-jail2]", "backend = polling", "filter =", "action =",
+				"[test-jail2]", "backend = polling", "filter =", 
+				"action =",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
-			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+			if DefLogSys.level <= logging.DEBUG:  # if DEBUG
 				_out_file(pjoin(cfg, "jail.conf"))
 
 		# create default test actions:
 		_write_action_cfg(actname="test-action1")
 		_write_action_cfg(actname="test-action2")
 
-		_write_jail_cfg(enabled=[1])
+		_write_jail_cfg(enabled=[1], actions=[1,2])
 		_write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3))
 		_write_file(test2log, "w")
 		_write_file(test3log, "w")
 		
 		# reload and wait for ban:
-		self.pruneLog("[test-phase 1]")
+		self.pruneLog("[test-phase 1a]")
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
@@ -738,11 +744,15 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				, MID_WAITTIME))
 		self.assertLogged("Added logfile: %r" % test1log)
 		self.assertLogged("[test-jail1] Ban 192.0.2.1")
+		# test actions started:
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action1: ** start'", 
+			"stdout: '[test-jail1] test-action2: ** start'", all=True)
 		
 		# enable both jails, 3 logs for jail1, etc...
 		# truncate test-log - we should not find unban/ban again by reload:
-		self.pruneLog("[test-phase 2a]")
-		_write_jail_cfg()
+		self.pruneLog("[test-phase 1b]")
+		_write_jail_cfg(actions=[1,2])
 		_write_file(test1log, "w+")
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
@@ -759,14 +769,47 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"Added logfile: %r" % test3log, all=True)
 		# test actions reloaded:
 		self.assertLogged(
-			"echo '[test-jail1] test-action1: ** reload' -- returned successfully", 
-			"echo '[test-jail1] test-action2: ** reload' -- returned successfully", all=True)
-
+			"stdout: '[test-jail1] test-action1: .. reload'", 
+			"stdout: '[test-jail1] test-action2: .. reload'", all=True)
 		# test 1 new jail:
 		self.assertLogged(
 			"Creating new jail 'test-jail2'",
 			"Jail 'test-jail2' started", all=True)
-
+		
+		# update action1, delete action2 (should be stopped via configuration)...
+		self.pruneLog("[test-phase 2a]")
+		_write_jail_cfg(actions=[1])
+		_write_action_cfg(actname="test-action1", 
+			start= "               echo '[<name>] %s: started.'" % "test-action1",
+			reload="               echo '[<name>] %s: reloaded.'" % "test-action1", 
+			stop=  "               echo '[<name>] %s: stopped.'" % "test-action1")
+		self.execSuccess(startparams, "reload")
+		self.assertTrue(
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		# test not unbanned / banned again:
+		self.assertNotLogged(
+			"[test-jail1] Unban 192.0.2.1", 
+			"[test-jail1] Ban 192.0.2.1", all=True)
+		# no new log files:
+		self.assertNotLogged("Added logfile:")
+		# test action reloaded (update):
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action1: .. reload'",
+			"stdout: '[test-jail1] test-action1: reloaded.'", all=True)
+		# test stopped action unbans:
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action2: -- unban 192.0.2.1'")
+		# test action stopped:
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action2: __ stop'")
+		self.assertNotLogged(
+			"stdout: '[test-jail1] test-action1: -- unban 192.0.2.1'")
+		
+		# don't need both actions anymore:
+		_write_action_cfg(actname="test-action1", allow=False)
+		_write_action_cfg(actname="test-action2", allow=False)
+		_write_jail_cfg(actions=[])
+		
 		# write new failures:
 		self.pruneLog("[test-phase 2b]")
 		_write_file(test2log, "w+", *(

From 4404642fa32d4626500d784e1569a8e94c949833 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 10:56:35 +0200
Subject: [PATCH 178/623] pyinotify-backend: stability fix for sporadically
 errors in multi-threaded environment (without lock)

---
 fail2ban/server/filterpyinotify.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index c5b5680b..1e08f250 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -119,14 +119,15 @@ class FilterPyinotify(FileFilter):
 		logSys.debug("Added file watcher for %s", path)
 
 	def _delFileWatcher(self, path):
-		wdInt = self.__watches[path]
-		wd = self.__monitor.rm_watch(wdInt)
-		if wd[wdInt]:
-			del self.__watches[path]
-			logSys.debug("Removed file watcher for %s", path)
-			return True
-		else:
-			return False
+		try:
+			wdInt = self.__watches.pop(path)
+			wd = self.__monitor.rm_watch(wdInt)
+			if wd[wdInt]:
+				logSys.debug("Removed file watcher for %s", path)
+				return True
+		except KeyError: # pragma: no cover
+			pass
+		return False
 
 	##
 	# Add a log file path
@@ -158,8 +159,11 @@ class FilterPyinotify(FileFilter):
 					if k.startswith(path_dir + pathsep)]):
 			# Remove watches for the directory
 			# since there is no other monitored file under this directory
-			wdInt = self.__watches.pop(path_dir)
-			self.__monitor.rm_watch(wdInt)
+			try:
+				wdInt = self.__watches.pop(path_dir)
+				self.__monitor.rm_watch(wdInt)
+			except KeyError: # pragma: no cover
+				pass
 			logSys.debug("Removed monitor for the parent directory %s", path_dir)
 
 	# pyinotify.ProcessEvent default handler:

From 2108216d33bc7edbbc5f40c3aaf9eac9e27279eb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 16:08:28 +0200
Subject: [PATCH 179/623] file filter-backends: stability fix for sporadically
 errors - always close file handle, otherwise may be locked (prevent
 log-rotate, etc.)

---
 fail2ban/server/filter.py | 75 +++++++++++++++++++--------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 6b0599f5..ab6e7b15 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -799,49 +799,48 @@ class FileFilter(Filter):
 		if log is None:
 			logSys.error("Unable to get failures in " + filename)
 			return False
-		# Try to open log file.
+		# We should always close log (file), otherwise may be locked (log-rotate, etc.)
 		try:
-			has_content = log.open()
-		# see http://python.org/dev/peps/pep-3151/
-		except IOError as e:
-			logSys.error("Unable to open %s" % filename)
-			logSys.exception(e)
-			return False
-		except OSError as e: # pragma: no cover - requires race condition to tigger this
-			logSys.error("Error opening %s" % filename)
-			logSys.exception(e)
-			return False
-		except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
-			logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
-			logSys.exception(e)
-			return False
-
-		# seek to find time for first usage only (prevent performance decline with polling of big files)
-		if self.__autoSeek.get(filename):
-			startTime = self.__autoSeek[filename]
-			del self.__autoSeek[filename]
-			# prevent completely read of big files first time (after start of service), 
-			# initial seek to start time using half-interval search algorithm:
+			# Try to open log file.
 			try:
-				self.seekToTime(log, startTime)
-			except Exception as e: # pragma: no cover
-				logSys.error("Error during seek to start time in \"%s\"", filename)
-				raise
+				has_content = log.open()
+			# see http://python.org/dev/peps/pep-3151/
+			except IOError as e:
+				logSys.error("Unable to open %s" % filename)
+				logSys.exception(e)
+				return False
+			except OSError as e: # pragma: no cover - requires race condition to tigger this
+				logSys.error("Error opening %s" % filename)
+				logSys.exception(e)
+				return False
+			except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
+				logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
 				logSys.exception(e)
 				return False
 
-		# yoh: has_content is just a bool, so do not expect it to
-		# change -- loop is exited upon break, and is not entered at
-		# all if upon container opening that one was empty.  If we
-		# start reading tested to be empty container -- race condition
-		# might occur leading at least to tests failures.
-		while has_content:
-			line = log.readline()
-			if not line or not self.active:
-				# The jail reached the bottom or has been stopped
-				break
-			self.processLineAndAdd(line)
-		log.close()
+			# seek to find time for first usage only (prevent performance decline with polling of big files)
+			if self.__autoSeek.get(filename):
+				startTime = self.__autoSeek[filename]
+				del self.__autoSeek[filename]
+				# prevent completely read of big files first time (after start of service), 
+				# initial seek to start time using half-interval search algorithm:
+				try:
+					self.seekToTime(log, startTime)
+				except Exception as e: # pragma: no cover
+					logSys.error("Error during seek to start time in \"%s\"", filename)
+					raise
+					logSys.exception(e)
+					return False
+
+			if has_content:
+				while not self.idle:
+					line = log.readline()
+					if not line or not self.active:
+						# The jail reached the bottom or has been stopped
+						break
+					self.processLineAndAdd(line)
+		finally:
+			log.close()
 		db = self.jail.database
 		if db is not None:
 			db.updateLog(self.jail, log)

From f6197200a9d250200e0f64fc26fd60c7e556ce5f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 16:12:48 +0200
Subject: [PATCH 180/623] introduced new flag "banned" as property, used to
 recognize the ticket was really banned; get/set restored flag functions
 rewritten to property "restored" similar to "banned"; several code
 optimizations and tests extensions;

---
 fail2ban/server/actions.py           | 21 +++++++++++++++++---
 fail2ban/server/banmanager.py        | 29 ++++++++++++++--------------
 fail2ban/server/jail.py              | 19 ++++++++++++++----
 fail2ban/server/ticket.py            | 22 ++++++++++++++++-----
 fail2ban/tests/banmanagertestcase.py | 17 ++++++++++++----
 fail2ban/tests/tickettestcase.py     | 18 +++++++++++++++++
 6 files changed, 96 insertions(+), 30 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 5ac7b9a9..bc8684b5 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -359,9 +359,10 @@ class Actions(JailThread, Mapping):
 				aInfo["ipjailmatches"]  = lambda: "\n".join(mi4ip().getMatches())
 				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
 				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
-			if self.__banManager.addBanTicket(bTicket):
+			reason = {}
+			if self.__banManager.addBanTicket(bTicket, reason=reason):
 				cnt += 1
-				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.getRestored() else 'Restore '), ip)
+				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
 						action.ban(aInfo.copy())
@@ -371,8 +372,22 @@ class Actions(JailThread, Mapping):
 							"info '%r': %s",
 							self._jail.name, name, aInfo, e,
 							exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+				# after all actions are processed set banned flag:
+				bTicket.banned = True
 			else:
-				logSys.notice("[%s] %s already banned", self._jail.name, ip)
+				bTicket = reason['ticket']
+				# if already banned (otherwise still process some action)
+				if bTicket.banned:
+					# compare time of failure occurrence with time ticket was really banned:
+					diftm = ticket.getTime() - bTicket.getTime()
+					# log already banned with following level:
+					#   DEBUG   - before 3 seconds - certain interval for it, because of possible latency by recognizing in backends, etc.
+					#   NOTICE  - before 60 seconds - may still occurre if action are slow, or very high load in backend,
+					#   WARNING - after 60 seconds - very long time, something may be wrong
+					ll = logging.DEBUG   if diftm < 3 \
+					else logging.NOTICE  if diftm < 60 \
+					else logging.WARNING
+					logSys.log(ll, "[%s] %s already banned", self._jail.name, ip)
 		if cnt:
 			logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, 
 				self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name)
diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 41e202ef..6da5366c 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -256,29 +256,30 @@ class BanManager:
 	# @param ticket the ticket
 	# @return True if the IP address is not in the ban list
 	
-	def addBanTicket(self, ticket):
+	def addBanTicket(self, ticket, reason={}):
+		eob = ticket.getEndOfBanTime(self.__banTime)
 		with self.__lock:
 			# check already banned
 			fid = ticket.getID()
 			oldticket = self.__banList.get(fid)
 			if oldticket:
-				# if already permanent
-				btold, told = oldticket.getBanTime(self.__banTime), oldticket.getTime()
-				if btold == -1:
-					return False
-				# if given time is less than already banned time
-				btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
-				if btnew != -1 and tnew + btnew <= told + btold:
-					return False
-				# we have longest ban - set new (increment) ban time
-				oldticket.setTime(tnew)
-				oldticket.setBanTime(btnew)
+				reason['ticket'] = oldticket
+				# if new time for end of ban is larger than already banned end-time:
+				if eob > oldticket.getEndOfBanTime(self.__banTime):
+					# we have longest ban - set new (increment) ban time
+					reason['prolong'] = 1
+					btm = ticket.getBanTime(self.__banTime)
+					# if not permanent:
+					if btm != -1:
+						diftm = ticket.getTime() - oldticket.getTime()
+						if diftm > 0:
+							btm += diftm
+					oldticket.setBanTime(btm)
 				return False
-			# not yet banned - add new
+			# not yet banned - add new one:
 			self.__banList[fid] = ticket
 			self.__banTotal += 1
 			# correct next unban time:
-			eob = ticket.getEndOfBanTime(self.__banTime)
 			if self.__nextUnbanTime > eob:
 				self.__nextUnbanTime = eob
 			return True
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index 7e05a025..e70eaddd 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -28,7 +28,7 @@ import Queue
 
 from .actions import Actions
 from ..client.jailreader import JailReader
-from ..helpers import getLogger
+from ..helpers import getLogger, MyTime
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -194,7 +194,7 @@ class Jail(object):
 		Used by filter to add a failure for banning.
 		"""
 		self.__queue.put(ticket)
-		if not ticket.getRestored() and self.database is not None:
+		if not ticket.restored and self.database is not None:
 			self.database.addBan(self, ticket)
 
 	def getFailTicket(self):
@@ -203,7 +203,8 @@ class Jail(object):
 		Used by actions to get a failure for banning.
 		"""
 		try:
-			return self.__queue.get(False)
+			ticket = self.__queue.get(False)
+			return ticket
 		except Queue.Empty:
 			return False
 
@@ -217,7 +218,17 @@ class Jail(object):
 					#logSys.debug('restored ticket: %s', ticket)
 					if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
 						# mark ticked was restored from database - does not put it again into db:
-						ticket.setRestored(True)
+						ticket.restored = True
+						# correct start time / ban time (by the same end of ban):
+						btm = ticket.getBanTime(forbantime)
+						diftm = MyTime.time() - ticket.getTime()
+						if btm != -1 and diftm > 0:
+							btm -= diftm
+						# ignore obsolete tickets:
+						if btm != -1 and btm <= 0:
+							continue
+						ticket.setTime(MyTime.time())
+						ticket.setBanTime(btm)
 						self.putFailTicket(ticket)
 		except Exception as e: # pragma: no cover
 			logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 8ff6a780..2f9a88ed 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -34,9 +34,10 @@ from .mytime import MyTime
 logSys = getLogger(__name__)
 
 
-class Ticket:
+class Ticket(object):
 	
 	RESTORED = 0x01
+	BANNED   = 0x08
 
 	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		"""Ticket constructor
@@ -135,14 +136,25 @@ class Ticket:
 	def getMatches(self):
 		return self._data.get('matches', [])
 
-	def setRestored(self, value):
+	@property
+	def restored(self):
+		return self._flags & Ticket.RESTORED
+	@restored.setter
+	def restored(self, value):
 		if value:
-			self._flags = Ticket.RESTORED
+			self._flags |= Ticket.RESTORED
 		else:
 			self._flags &= ~(Ticket.RESTORED)
 	
-	def getRestored(self):
-		return self._flags & Ticket.RESTORED
+	@property
+	def banned(self):
+		return self._flags & Ticket.BANNED
+	@banned.setter
+	def banned(self, value):
+		if value:
+			self._flags |= Ticket.BANNED
+		else:
+			self._flags &= ~(Ticket.BANNED)
 
 	def setData(self, *args, **argv):
 		# if overwrite - set data and filter None values:
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index c9f41c6c..4a86102c 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -53,11 +53,15 @@ class AddFailure(unittest.TestCase):
 		self.assertEqual(self.__banManager.size(), 1)
 
 	def testAddDuplicateWithTime(self):
+		defBanTime = self.__banManager.getBanTime()
+		prevEndOfBanTime = 0
 		# add again a duplicate :
-		#   1) with newer start time and the same ban time
+		#   0) with same start time and the same (default) ban time
+		#   1) with newer start time and the same (default) ban time
 		#   2) with same start time and longer ban time
     #   3) with permanent ban time (-1)
 		for tnew, btnew in (
+			(1167605999.0,       None),
 			(1167605999.0 + 100, None),
 			(1167605999.0,       24*60*60),
 			(1167605999.0,       -1),
@@ -71,9 +75,14 @@ class AddFailure(unittest.TestCase):
 			self.assertEqual(self.__banManager.size(), 1)
 			# pop ticket and check it was prolonged :
 			banticket = self.__banManager.getTicketByID(ticket2.getID())
-			self.assertEqual(banticket.getTime(), ticket2.getTime())
-			self.assertEqual(banticket.getTime(), ticket2.getTime())
-			self.assertEqual(banticket.getBanTime(), ticket2.getBanTime(self.__banManager.getBanTime()))
+			self.assertEqual(banticket.getEndOfBanTime(defBanTime), ticket2.getEndOfBanTime(defBanTime))
+			self.assertTrue(banticket.getEndOfBanTime(defBanTime) > prevEndOfBanTime)
+			prevEndOfBanTime = ticket1.getEndOfBanTime(defBanTime)
+			# but the start time should not be changed (+ 100 is ignored):
+			self.assertEqual(banticket.getTime(), 1167605999.0)
+			# if prolong to permanent, it should also have permanent ban time:
+			if btnew == -1:
+				self.assertEqual(banticket.getBanTime(defBanTime), -1)
 
 	def testInListOK(self):
 		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py
index 68a44bb5..277c2f28 100644
--- a/fail2ban/tests/tickettestcase.py
+++ b/fail2ban/tests/tickettestcase.py
@@ -108,6 +108,24 @@ class TicketTests(unittest.TestCase):
     self.assertEqual(ft2.getLastTime(), ft.getLastTime())
     self.assertEqual(ft2.getBanTime(), ft.getBanTime())
 
+  def testTicketFlags(self):
+    flags = ('restored', 'banned')
+    ticket = Ticket('test', 0)
+    trueflags = []
+    for v in (True, False, True):
+      for f in flags:
+        setattr(ticket, f, v)
+        if v:
+          trueflags.append(f)
+        else:
+          trueflags.remove(f)
+        for f2 in flags:
+          self.assertEqual(bool(getattr(ticket, f2)), f2 in trueflags)
+    ## inherite props from another tockets:
+    ticket = FailTicket(ticket=ticket)
+    for f2 in flags:
+      self.assertTrue(bool(getattr(ticket, f2)))
+
   def testTicketData(self):
     t = BanTicket('193.168.0.128', None, ['first', 'second'])
     # expand data (no overwrites, matches are available) :

From a20f325f80489d392902751db7f30fd67aa393bb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 17:36:01 +0200
Subject: [PATCH 181/623] database: stability fix - repack cursor iterator as
 long as locked

---
 fail2ban/server/database.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 24445f3d..639331bf 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -482,7 +482,8 @@ class Fail2BanDb(object):
 			queryArgs.append(str(ip))
 		query += " ORDER BY ip, timeofban desc"
 
-		return cur.execute(query, queryArgs)
+		# repack iterator as long as in lock:
+		return list(cur.execute(query, queryArgs))
 
 	def getBans(self, **kwargs):
 		"""Get bans from the database.
@@ -581,7 +582,6 @@ class Fail2BanDb(object):
 				self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 			return tickets if ip is None else ticket
 
-	@commitandrollback
 	def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None):
 		if fromtime is None:
 			fromtime = MyTime.time()
@@ -606,7 +606,9 @@ class Fail2BanDb(object):
 		tickets = []
 		ticket = None
 
-		results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
+		with self._lock:
+			results = list(self._getCurrentBans(self._db.cursor(), 
+				jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
 
 		if results:
 			for banip, timeofban, data in results:

From e0347bb3a03175a740fe7654ba2df0dc562c09fa Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 17:50:25 +0200
Subject: [PATCH 182/623] assertLogged extended with parameter wait (to wait up
 to specified timeout, before we throw assert exception) + test cases
 rewritten using that

---
 fail2ban/tests/fail2banclienttestcase.py | 58 +++++++++---------------
 fail2ban/tests/utils.py                  | 37 +++++++++++----
 2 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 2804fcd4..8159a9f6 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -737,11 +737,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
-		self.assertTrue(
-			Utils.wait_for(lambda: \
-				self._is_logged("Reload finished.") and
-				self._is_logged("1 ticket(s) in 'test-jail1")
-				, MID_WAITTIME))
+		self.assertLogged(
+			"Reload finished.",
+			"1 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME)
 		self.assertLogged("Added logfile: %r" % test1log)
 		self.assertLogged("[test-jail1] Ban 192.0.2.1")
 		# test actions started:
@@ -757,8 +755,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
-		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test not unbanned / banned again:
 		self.assertNotLogged(
 			"[test-jail1] Unban 192.0.2.1", 
@@ -784,8 +781,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			reload="               echo '[<name>] %s: reloaded.'" % "test-action1", 
 			stop=  "               echo '[<name>] %s: stopped.'" % "test-action1")
 		self.execSuccess(startparams, "reload")
-		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test not unbanned / banned again:
 		self.assertNotLogged(
 			"[test-jail1] Unban 192.0.2.1", 
@@ -821,11 +817,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test2log)
 		# test all will be found in jail1 and one in jail2:
-		self.assertTrue(
-			Utils.wait_for(lambda: \
-				self._is_logged("2 ticket(s) in 'test-jail2") and
-				self._is_logged("5 ticket(s) in 'test-jail1")
-				, MID_WAITTIME))
+		self.assertLogged(
+			"2 ticket(s) in 'test-jail2",
+			"5 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME)
 		self.assertLogged(
 			"[test-jail1] Ban 192.0.2.2",
 			"[test-jail1] Ban 192.0.2.3",
@@ -848,12 +842,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 2c]")
 		self.execSuccess(startparams,
 			"restart", "test-jail2")
-		self.assertTrue(
-			Utils.wait_for(lambda: \
-				self._is_logged("Reload finished.") and
-				self._is_logged("Restore Ban") and
-				self._is_logged("2 ticket(s) in 'test-jail2")
-				, MID_WAITTIME))
+		self.assertLogged(
+			"Reload finished.",
+			"Restore Ban",
+			"2 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME)
 		# stop/start and unban/restore ban:
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
@@ -868,11 +860,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 2d]")
 		self.execSuccess(startparams,
 			"restart", "--unban", "test-jail2")
-		self.assertTrue(
-			Utils.wait_for(lambda: \
-				self._is_logged("Reload finished.") and
-				self._is_logged("Jail 'test-jail2' started")
-				, MID_WAITTIME))
+		self.assertLogged(
+			"Reload finished.",
+			"Jail 'test-jail2' started", all=True, wait=MID_WAITTIME)
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
@@ -888,8 +878,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
 		self.execSuccess(startparams, "reload", "test-jail1")
-		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		self.assertLogged(
+			"Reload finished.", all=True, wait=MID_WAITTIME)
 		self.assertLogged(
 			"Reload jail 'test-jail1'",
 			"Jail 'test-jail1' reloaded", all=True)
@@ -903,8 +893,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 4]")
 		_write_jail_cfg(enabled=[1])
 		self.execSuccess(startparams, "reload")
-		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test both jails should be reloaded:
 		self.assertLogged(
 			"Reload jail 'test-jail1'")
@@ -926,11 +915,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		))
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test1log)
-		self.assertTrue(
-			Utils.wait_for(lambda: \
-				self._is_logged("6 ticket(s) in 'test-jail1") and
-				self._is_logged("[test-jail1] 192.0.2.1 already banned")
-				, MID_WAITTIME))
+		self.assertLogged(
+			"6 ticket(s) in 'test-jail1",
+			"[test-jail1] 192.0.2.1 already banned", all=True, wait=MID_WAITTIME)
 		# test "failure" regexp still available:
 		self.assertLogged(
 			"[test-jail1] Found 192.0.2.1",
@@ -953,8 +940,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 7]")
 		self.execSuccess(startparams,
 			"reload", "--unban")
-		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# reloads unbanned all:
 		self.assertLogged(
 			"Jail 'test-jail1' reloaded",
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 6eaf560b..76bb724c 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -589,8 +589,21 @@ class LogCaptureTestCase(unittest.TestCase):
 		logSys.handlers = self._old_handlers
 		logSys.level = self._old_level
 
-	def _is_logged(self, s):
-		return s in self._log.getvalue()
+	def _is_logged(self, *s, **kwargs):
+		logged = self._log.getvalue()
+		if not kwargs.get('all', False):
+			# at least one entry should be found:
+			for s_ in s:
+				if s_ in logged:
+					return True
+			if True: # pragma: no cover
+				return False
+		else:
+			# each entry should be found:
+			for s_ in s:
+				if s_ not in logged: # pragma: no cover
+					return False
+			return True
 
 	def assertLogged(self, *s, **kwargs):
 		"""Assert that one of the strings was logged
@@ -604,19 +617,23 @@ class LogCaptureTestCase(unittest.TestCase):
 		  Test should succeed if string (or any of the listed) is present in the log
 		all : boolean (default False) if True should fail if any of s not logged
 		"""
-		logged = self._log.getvalue()
+		wait = kwargs.get('wait', None)
+		if wait:
+			res = Utils.wait_for(lambda: self._is_logged(*s, **kwargs), wait)
+		else:
+			res = self._is_logged(*s, **kwargs)
 		if not kwargs.get('all', False):
 			# at least one entry should be found:
-			for s_ in s:
-				if s_ in logged:
-					return
-			if True: # pragma: no cover
+			if not res: # pragma: no cover
+				logged = self._log.getvalue()
 				self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
 		else:
 			# each entry should be found:
-			for s_ in s:
-				if s_ not in logged: # pragma: no cover
-					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
+			if not res: # pragma: no cover
+				logged = self._log.getvalue()
+				for s_ in s:
+					if s_ not in logged:
+						self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
 	def assertNotLogged(self, *s, **kwargs):
 		"""Assert that strings were not logged

From ebd864660a4377a4af3fa77793517db4b97f3833 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 18:28:47 +0200
Subject: [PATCH 183/623] normalize usage of preferred encoding (and decode any
 to string); python 3.x compatibility (used uni_decode for string
 representation of stdout/stderr, unified test cases) amend for #1542

---
 fail2ban/client/fail2banregex.py |  5 ++---
 fail2ban/helpers.py              | 34 ++++++++++++++++++++++++++++++++
 fail2ban/server/database.py      | 13 ++++++------
 fail2ban/server/filter.py        | 30 +++-------------------------
 fail2ban/server/filtersystemd.py | 16 ++++++---------
 fail2ban/server/utils.py         | 14 +++++++++----
 fail2ban/tests/actiontestcase.py |  4 ++--
 fail2ban/tests/filtertestcase.py |  7 +++----
 fail2ban/tests/misctestcase.py   | 10 ++++------
 fail2ban/tests/servertestcase.py |  5 ++---
 10 files changed, 72 insertions(+), 66 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index f36ca7b0..ea2f3a7f 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -29,7 +29,6 @@ __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halch
 __license__ = "GPL"
 
 import getopt
-import locale
 import logging
 import os
 import shlex
@@ -52,7 +51,7 @@ from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import RegexException
 
-from ..helpers import FormatterWithTraceBack, getLogger
+from ..helpers import FormatterWithTraceBack, getLogger, PREFER_ENC
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
@@ -239,7 +238,7 @@ class Fail2banRegex(object):
 		if opts.encoding:
 			self.encoding = opts.encoding
 		else:
-			self.encoding = locale.getpreferredencoding()
+			self.encoding = PREFER_ENC
 		self.raw = True if opts.raw else False
 
 	def decode_line(self, line):
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 7660e64a..2738f5e6 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -21,6 +21,7 @@ __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko"
 __license__ = "GPL"
 
 import gc
+import locale
 import logging
 import os
 import re
@@ -32,6 +33,9 @@ from threading import Lock
 from .server.mytime import MyTime
 
 
+PREFER_ENC = locale.getpreferredencoding()
+
+
 def formatExceptionInfo():
 	""" Consistently format exception information """
 	cla, exc = sys.exc_info()[:2]
@@ -144,6 +148,36 @@ def splitwords(s):
 	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
 
 
+#
+# Following "uni_decode" function unified python independent any to string converting
+#
+# Typical example resp. work-case for understanding the coding/decoding issues:
+#
+#   [isinstance('', str), isinstance(b'', str), isinstance(u'', str)]
+#   [True, True, False]; # -- python2
+#	  [True, False, True]; # -- python3
+#
+if sys.version_info >= (3,):
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, bytes):
+				return x.decode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict': 
+				raise
+			return uni_decode(x, enc, 'replace')
+else:
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, unicode):
+				return x.encode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict':
+				raise
+			return uni_decode(x, enc, 'replace')
+
 class BgService(object):
 	"""Background servicing
 
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 639331bf..120619f9 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -22,7 +22,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
 import json
-import locale
 import shutil
 import sqlite3
 import sys
@@ -32,7 +31,7 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -41,7 +40,7 @@ if sys.version_info >= (3,):
 	def _json_dumps_safe(x):
 		try:
 			x = json.dumps(x, ensure_ascii=False).encode(
-				locale.getpreferredencoding(), 'replace')
+				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
@@ -50,7 +49,7 @@ if sys.version_info >= (3,):
 	def _json_loads_safe(x):
 		try:
 			x = json.loads(x.decode(
-				locale.getpreferredencoding(), 'replace'))
+				PREFER_ENC, 'replace'))
 		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
@@ -62,14 +61,14 @@ else:
 		elif isinstance(x, list):
 			return [_normalize(element) for element in x]
 		elif isinstance(x, unicode):
-			return x.encode(locale.getpreferredencoding())
+			return x.encode(PREFER_ENC)
 		else:
 			return x
 
 	def _json_dumps_safe(x):
 		try:
 			x = json.dumps(_normalize(x), ensure_ascii=False).decode(
-				locale.getpreferredencoding(), 'replace')
+				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
@@ -78,7 +77,7 @@ else:
 	def _json_loads_safe(x):
 		try:
 			x = _normalize(json.loads(x.decode(
-				locale.getpreferredencoding(), 'replace')))
+				PREFER_ENC, 'replace')))
 		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index ab6e7b15..08f3e77d 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -24,7 +24,6 @@ __license__ = "GPL"
 import codecs
 import datetime
 import fcntl
-import locale
 import logging
 import os
 import re
@@ -40,7 +39,7 @@ from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -87,7 +86,7 @@ class Filter(JailThread):
 		## External command
 		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
-		self.__encoding = locale.getpreferredencoding()
+		self.__encoding = PREFER_ENC
 		## Error counter (protected, so can be used in filter implementations)
 		## if it reached 100 (at once), run-cycle will go idle
 		self._errors = 0
@@ -329,7 +328,7 @@ class Filter(JailThread):
 
 	def setLogEncoding(self, encoding):
 		if encoding.lower() == "auto":
-			encoding = locale.getpreferredencoding()
+			encoding = PREFER_ENC
 		codecs.lookup(encoding) # Raise LookupError if invalid codec
 		self.__encoding = encoding
 		logSys.info("  encoding: %s" % encoding)
@@ -452,29 +451,6 @@ class Filter(JailThread):
 
 		return False
 
-	if sys.version_info >= (3,):
-		@staticmethod
-		def uni_decode(x, enc, errors='strict'):
-			try:
-				if isinstance(x, bytes):
-					return x.decode(enc, errors)
-				return x
-			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-				if errors != 'strict': 
-					raise
-				return uni_decode(x, enc, 'replace')
-	else:
-		@staticmethod
-		def uni_decode(x, enc, errors='strict'):
-			try:
-				if isinstance(x, unicode):
-					return x.encode(enc, errors)
-				return x
-			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-				if errors != 'strict':
-					raise
-				return uni_decode(x, enc, 'replace')
-
 	def processLine(self, line, date=None, returnRawHost=False,
 		checkAllRegex=False, checkFindTime=False):
 		"""Split the time portion from log msg and return findFailures on them
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 0b6c4074..f9f9395c 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -34,7 +34,7 @@ from .failmanager import FailManagerEmpty
 from .filter import JournalFilter, Filter
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, logging, splitwords
+from ..helpers import getLogger, logging, splitwords, uni_decode
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -174,10 +174,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def getJournalMatch(self):
 		return self.__matches
 
-	def uni_decode(self, x):
-		v = Filter.uni_decode(x, self.getLogEncoding())
-		return v
-
 	##
 	# Format journal log entry into syslog style
 	#
@@ -186,16 +182,16 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 	def formatJournalEntry(self, logentry):
 		# Be sure, all argument of line tuple should have the same type:
-		uni_decode = self.uni_decode
+		enc = self.getLogEncoding()
 		logelements = []
 		v = logentry.get('_HOSTNAME')
 		if v:
-			logelements.append(uni_decode(v))
+			logelements.append(uni_decode(v, enc))
 		v = logentry.get('SYSLOG_IDENTIFIER')
 		if not v:
 			v = logentry.get('_COMM')
 		if v:
-			logelements.append(uni_decode(v))
+			logelements.append(uni_decode(v, enc))
 			v = logentry.get('SYSLOG_PID')
 			if not v:
 				v = logentry.get('_PID')
@@ -210,9 +206,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				logelements.append("[%12.6f]" % monotonic.total_seconds())
 		msg = logentry.get('MESSAGE','')
 		if isinstance(msg, list):
-			logelements.append(" ".join(uni_decode(v) for v in msg))
+			logelements.append(" ".join(uni_decode(v, enc) for v in msg))
 		else:
-			logelements.append(uni_decode(msg))
+			logelements.append(uni_decode(msg, enc))
 
 		logline = " ".join(logelements)
 
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 738394c3..a8a00257 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -21,8 +21,14 @@ __author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester"
 __license__ = "GPL"
 
-import logging, os, fcntl, subprocess, time, signal
-from ..helpers import getLogger
+import fcntl
+import logging
+import os
+import signal
+import subprocess
+import sys
+import time
+from ..helpers import getLogger, uni_decode
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -179,7 +185,7 @@ class Utils():
 				if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
 					logSys.log(std_level, "%s -- stdout:", realCmd)
 					for l in stdout.splitlines():
-						logSys.log(std_level, " -- stdout: %r", l)
+						logSys.log(std_level, " -- stdout: %r", uni_decode(l))
 				popen.stdout.close()
 			if popen.stderr:
 				try:
@@ -191,7 +197,7 @@ class Utils():
 				if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
 					logSys.log(std_level, "%s -- stderr:", realCmd)
 					for l in stderr.splitlines():
-						logSys.log(std_level, " -- stderr: %r", l)
+						logSys.log(std_level, " -- stderr: %r", uni_decode(l))
 				popen.stderr.close()
 
 		success = False
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 166450bb..cacb5e17 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -399,11 +399,11 @@ class CommandActionTest(LogCaptureTestCase):
 
 	def testCaptureStdOutErr(self):
 		CommandAction.executeCmd('echo "How now brown cow"')
-		self.assertLogged("stdout: 'How now brown cow'\n", "stdout: b'How now brown cow'\n")
+		self.assertLogged("stdout: 'How now brown cow'\n")
 		CommandAction.executeCmd(
 			'echo "The rain in Spain stays mainly in the plain" 1>&2')
 		self.assertLogged(
-			"stderr: 'The rain in Spain stays mainly in the plain'\n", "stderr: b'The rain in Spain stays mainly in the plain'\n")
+			"stderr: 'The rain in Spain stays mainly in the plain'\n")
 
 	def testCallingMap(self):
 		mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'),
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index f7394a8e..15495655 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -38,11 +38,11 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, locale
+from ..server.filter import Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
 from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
-from ..server.utils import Utils
+from ..server.utils import Utils, uni_decode
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
 from .dummyjail import DummyJail
 
@@ -311,8 +311,7 @@ class BasicFilter(unittest.TestCase):
 					b'Fail for "g\xc3\xb6ran" from 192.0.2.1'
 				):
 					# join should work if all arguments have the same type:
-					enc = locale.getpreferredencoding()
-					"".join([Filter.uni_decode(v, enc) for v in (a1, a2, a3)])
+					"".join([uni_decode(v) for v in (a1, a2, a3)])
 
 
 class IgnoreIP(LogCaptureTestCase):
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 5b6b93a6..8fb393ea 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -35,7 +35,7 @@ from StringIO import StringIO
 
 from utils import LogCaptureTestCase, logSys as DefLogSys
 
-from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger
+from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, uni_decode
 from ..helpers import splitwords
 from ..server.datedetector import DateDetector
 from ..server.datetemplate import DatePatternRegex
@@ -74,16 +74,14 @@ class HelpersTest(unittest.TestCase):
 
 if sys.version_info >= (2,7):
 	def _sh_call(cmd):
-		import subprocess, locale
+		import subprocess
 		ret = subprocess.check_output(cmd, shell=True)
-		if sys.version_info >= (3,):
-			ret = ret.decode(locale.getpreferredencoding(), 'replace')
-		return str(ret).rstrip()
+		return uni_decode(ret).rstrip()
 else:
 	def _sh_call(cmd):
 		import subprocess
 		ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
-		return str(ret).rstrip()
+		return uni_decode(ret).rstrip()
 
 def _getSysPythonVersion():
 	return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'")
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 3ca8049b..d044da4b 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -28,7 +28,6 @@ import unittest
 import time
 import tempfile
 import os
-import locale
 import sys
 import platform
 
@@ -40,7 +39,7 @@ from ..server.jail import Jail
 from ..server.jailthread import JailThread
 from ..server.utils import Utils
 from .utils import LogCaptureTestCase
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 from .. import version
 
 try:
@@ -354,7 +353,7 @@ class Transmitter(TransmitterBase):
 	def testJailLogEncoding(self):
 		self.setGetTest("logencoding", "UTF-8", jail=self.jailName)
 		self.setGetTest("logencoding", "ascii", jail=self.jailName)
-		self.setGetTest("logencoding", "auto", locale.getpreferredencoding(),
+		self.setGetTest("logencoding", "auto", PREFER_ENC,
 			jail=self.jailName)
 		self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)
 

From 20b92f3ead34fe34f4d97077e73d6edaa80e0244 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Sep 2016 20:31:52 +0200
Subject: [PATCH 184/623] fail2ban-regex: build replacement of `<HOST>`
 substitution corresponding parameter `usedns` - now also in fail2ban-regex
 (amend)

---
 fail2ban/client/fail2banregex.py        | 5 +++++
 fail2ban/tests/fail2banregextestcase.py | 9 +++++++++
 2 files changed, 14 insertions(+)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index ea2f3a7f..f99f9f01 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -126,6 +126,9 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   help="File encoding. Default: system locale"),
 		Option("-r", "--raw", action='store_true',
 			   help="Raw hosts, don't resolve dns"),
+		Option("--usedns", action='store', default=None,
+			   help="DNS specified replacement of tags <HOST> in regexp "
+			        "('yes' - matches all form of hosts, 'no' - IP addresses only)"),
 		Option("-L", "--maxlines", type=int, default=0,
 			   help="maxlines for multi-line regex"),
 		Option("-m", "--journalmatch",
@@ -240,6 +243,8 @@ class Fail2banRegex(object):
 		else:
 			self.encoding = PREFER_ENC
 		self.raw = True if opts.raw else False
+		if opts.usedns:
+			self._filter.setUseDns(opts.usedns)
 
 	def decode_line(self, line):
 		return FileContainer.decode_line('<LOG>', self.encoding, line)
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 1d38a58e..0154e5af 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -131,6 +131,15 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(fail2banRegex.start(opts, args))
 		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
 
+	def testDirectRE_1raw_noDns(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--print-all-matched", "--raw", "--usedns=no",
+			Fail2banRegexTest.FILENAME_01, 
+			Fail2banRegexTest.RE_00
+		)
+		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed')
+
 	def testDirectRE_2(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"--print-all-matched",

From 35ce1166b61a69c0216c6880b166d0d3bad2b8a7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 21 Sep 2016 09:17:56 +0200
Subject: [PATCH 185/623] allows to update some configuration options (read
 with config-readers) with command line option, e. g.: ```bash ## start server
 with DEBUG log-level (ignore level read from fail2ban.conf): fail2ban-client
 --loglevel DEBUG start ## or fail2ban-server -c /cfg/path --loglevel DEBUG
 start ## keep server log-level by reload (without restart it) fail2ban-client
 --loglevel DEBUG reload ## switch log-level back to INFO: fail2ban-client set
 loglevel INFO ```

---
 fail2ban/client/configurator.py    | 4 ++--
 fail2ban/client/fail2bancmdline.py | 2 +-
 fail2ban/client/fail2banreader.py  | 4 +++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py
index 3b9845b6..4d28238f 100644
--- a/fail2ban/client/configurator.py
+++ b/fail2ban/client/configurator.py
@@ -72,8 +72,8 @@ class Configurator:
 	def getEarlyOptions(self):
 		return self.__fail2ban.getEarlyOptions()
 
-	def getOptions(self, jail = None):
-		self.__fail2ban.getOptions()
+	def getOptions(self, jail=None, updateMainOpt=None):
+		self.__fail2ban.getOptions(updateMainOpt)
 		return self.__jails.getOptions(jail)
 		
 	def convertToProtocol(self):
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 356255af..e1bc5f1e 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -242,7 +242,7 @@ class Fail2banCmdLine():
 		try:
 			self.configurator.Reload()
 			self.configurator.readAll()
-			ret = self.configurator.getOptions(jail)
+			ret = self.configurator.getOptions(jail, self._conf)
 			self.configurator.convertToProtocol()
 			stream = self.configurator.getConfigStream()
 		except Exception as e:
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index b3012c9b..8e1b0e5c 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -49,13 +49,15 @@ class Fail2banReader(ConfigReader):
 		]
 		return ConfigReader.getOptions(self, "Definition", opts)
 	
-	def getOptions(self):
+	def getOptions(self, updateMainOpt=None):
 		opts = [["string", "loglevel", "INFO" ],
 				["string", "logtarget", "STDERR"],
 				["string", "syslogsocket", "auto"],
 				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
 				["string", "dbpurgeage", "1d"]]
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
+		if updateMainOpt:
+			self.__opts.update(updateMainOpt)
 	
 	def convert(self):
 		# Ensure logtarget/level set first so any db errors are captured

From d153555a07ec81dabe4678ea06c8e644afc8ec40 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 14:28:47 +0200
Subject: [PATCH 186/623] increase default wait operation (sleep time,
 threshold interval) - avowedly greater inertance, but fewer system load by
 many jails resp. log files; waiting with `wait_for` extended with verifying
 of active flag; implemented better error handling in some multi-threaded
 routines; shutdown of jails rewritten (faster and safer, does not breaks
 shutdown process if some error occurred);

---
 fail2ban/server/actions.py         |  5 +-
 fail2ban/server/asyncserver.py     | 39 ++++++++------
 fail2ban/server/filter.py          | 20 ++++---
 fail2ban/server/filtergamin.py     |  5 +-
 fail2ban/server/filterpoll.py      |  7 +--
 fail2ban/server/filterpyinotify.py |  2 +-
 fail2ban/server/filtersystemd.py   | 83 +++++++++++++++++++-----------
 fail2ban/server/jail.py            | 24 ++++++---
 fail2ban/server/server.py          | 20 ++++---
 fail2ban/server/utils.py           |  4 +-
 10 files changed, 130 insertions(+), 79 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index bc8684b5..eed481d7 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -281,9 +281,10 @@ class Actions(JailThread, Mapping):
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
 			if self.idle:
-				time.sleep(self.sleeptime)
+				Utils.wait_for(lambda: not self.active or not self.idle,
+					self.sleeptime * 10, self.sleeptime)
 				continue
-			if not Utils.wait_for(self.__checkBan, self.sleeptime):
+			if not Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime):
 				self.__checkUnBan()
 		
 		self.__flushBan()
diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index ffb20503..a489b585 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -67,23 +67,28 @@ class RequestHandler(asynchat.async_chat):
 	# This method is called once we have a complete request.
 
 	def found_terminator(self):
-		# Pop whole buffer
-		message = self.__buffer
-		self.__buffer = []		
-		# Joins the buffer items.
-		message = CSPROTO.EMPTY.join(message)
-		# Closes the channel if close was received
-		if message == CSPROTO.CLOSE:
-			self.close_when_done()
-			return
-		# Deserialize
-		message = loads(message)
-		# Gives the message to the transmitter.
-		message = self.__transmitter.proceed(message)
-		# Serializes the response.
-		message = dumps(message, HIGHEST_PROTOCOL)
-		# Sends the response to the client.
-		self.push(message + CSPROTO.END)
+		try:
+			# Pop whole buffer
+			message = self.__buffer
+			self.__buffer = []		
+			# Joins the buffer items.
+			message = CSPROTO.EMPTY.join(message)
+			# Closes the channel if close was received
+			if message == CSPROTO.CLOSE:
+				self.close_when_done()
+				return
+			# Deserialize
+			message = loads(message)
+			# Gives the message to the transmitter.
+			message = self.__transmitter.proceed(message)
+			# Serializes the response.
+			message = dumps(message, HIGHEST_PROTOCOL)
+			# Sends the response to the client.
+			self.push(message + CSPROTO.END)
+		except Exception as e: # pragma: no cover
+			logSys.error("Caught unhandled exception: %r", e,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+
 		
 	def handle_error(self):
 		e1, e2 = formatExceptionInfo()
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 08f3e77d..ac64fb86 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -501,14 +501,18 @@ class Filter(JailThread):
 		except Exception as e:
 			logSys.error("Failed to process line: %r, caught exception: %r", line, e,
 				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-			# incr error counter, stop processing (going idle) after 100th error :
-			self._errors += 1
-			# sleep a little bit (to get around time-related errors):
-			time.sleep(self.sleeptime)
-			if self._errors >= 100:
-				logSys.error("Too many errors at once (%s), going idle", self._errors)
-				self._errors //= 2
-				self.idle = True
+			# incr common error counter:
+			self.commonError()
+
+	def commonError(self):
+		# incr error counter, stop processing (going idle) after 100th error :
+		self._errors += 1
+		# sleep a little bit (to get around time-related errors):
+		time.sleep(self.sleeptime)
+		if self._errors >= 100:
+			logSys.error("Too many errors at once (%s), going idle", self._errors)
+			self._errors //= 2
+			self.idle = True
 
 	##
 	# Returns true if the line should be ignored.
diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index 39793067..a8d03ebd 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -124,12 +124,13 @@ class FilterGamin(FileFilter):
 		while self.active:
 			if self.idle:
 				# wait a little bit here for not idle, to prevent hi-load:
-				if not Utils.wait_for(lambda: not self.idle, 
+				if not Utils.wait_for(lambda: not self.active or not self.idle,
 					self.sleeptime * 10, self.sleeptime
 				):
 					self.ticks += 1
 					continue
-			Utils.wait_for(self._handleEvents, self.sleeptime)
+			Utils.wait_for(lambda: not self.active or self._handleEvents(),
+				self.sleeptime)
 			self.ticks += 1
 		logSys.debug(self.jail.name + ": filter terminated")
 		return True
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index f66be42b..d0660f15 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -101,14 +101,15 @@ class FilterPoll(FileFilter):
 				logSys.log(6, "Woke up idle=%s with %d files monitored",
 						   self.idle, self.getLogCount())
 			if self.idle:
-				if not Utils.wait_for(lambda: not self.idle, 
+				if not Utils.wait_for(lambda: not self.active or not self.idle, 
 					self.sleeptime * 10, self.sleeptime
 				):
 					self.ticks += 1
 					continue
 			# Get file modification
 			modlst = []
-			Utils.wait_for(lambda: self.getModified(modlst), self.sleeptime)
+			Utils.wait_for(lambda: not self.active or self.getModified(modlst),
+				self.sleeptime)
 			for filename in modlst:
 				self.getFailures(filename)
 				self.__modified = True
@@ -162,7 +163,7 @@ class FilterPoll(FileFilter):
 							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
-			self._errors += 1
+			self.commonError()
 			if self.__file404Cnt[filename] > 50:
 				logSys.warning("Too many errors. Remove file %r from monitoring process", filename)
 				self.__file404Cnt[filename] = 0
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 1e08f250..9b444844 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -178,7 +178,7 @@ class FilterPyinotify(FileFilter):
 	# slow check events while idle:
 	def __check_events(self, *args, **kwargs):
 		if self.idle:
-			if Utils.wait_for(lambda: not self.idle, 
+			if Utils.wait_for(lambda: not self.active or not self.idle,
 				self.sleeptime * 10, self.sleeptime
 			):
 				pass
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index f9f9395c..a29085ba 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -252,40 +252,63 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 		while self.active:
 			# wait for records (or for timeout in sleeptime seconds):
-			self.__journal.wait(self.sleeptime)
-			if self.idle:
-				# because journal.wait will returns immediatelly if we have records in journal,
-				# just wait a little bit here for not idle, to prevent hi-load:
-				if not Utils.wait_for(lambda: not self.idle, 
-					self.sleeptime * 10, self.sleeptime
-				):
+			try:
+				## todo: find better method as wait_for to break (e.g. notify) journal.wait(self.sleeptime),
+				## don't use `journal.close()` for it, because in some python/systemd implementation it may 
+				## cause abnormal program termination
+				#self.__journal.wait(self.sleeptime) != journal.NOP
+				## 
+				## wait for entries without sleep in intervals, because "sleeping" in journal.wait:
+				Utils.wait_for(lambda: not self.active or \
+					self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP,
+					self.sleeptime, 0.00001)
+				if self.idle:
+					# because journal.wait will returns immediatelly if we have records in journal,
+					# just wait a little bit here for not idle, to prevent hi-load:
+					if not Utils.wait_for(lambda: not self.active or not self.idle, 
+						self.sleeptime * 10, self.sleeptime
+					):
+						self.ticks += 1
+						continue
+				self.__modified = 0
+				while self.active:
+					logentry = None
+					try:
+						logentry = self.__journal.get_next()
+					except OSError as e:
+						logSys.error("Error reading line from systemd journal: %s",
+							e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
 					self.ticks += 1
-					continue
-			self.__modified = 0
-			while self.active:
-				logentry = None
-				try:
-					logentry = self.__journal.get_next()
-				except OSError as e:
-					logSys.error("Error reading line from systemd journal: %s",
-						e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
-				self.ticks += 1
-				if logentry:
-					self.processLineAndAdd(
-						*self.formatJournalEntry(logentry))
-					self.__modified += 1
-					if self.__modified >= 100: # todo: should be configurable
+					if logentry:
+						self.processLineAndAdd(
+							*self.formatJournalEntry(logentry))
+						self.__modified += 1
+						if self.__modified >= 100: # todo: should be configurable
+							break
+					else:
 						break
-				else:
+				if self.__modified:
+					try:
+						while True:
+							ticket = self.failManager.toBan()
+							self.jail.putFailTicket(ticket)
+					except FailManagerEmpty:
+						self.failManager.cleanup(MyTime.time())
+			except Exception as e: # pragma: no cover
+				if not self.active: # if not active - error by stop...
 					break
-			if self.__modified:
-				try:
-					while True:
-						ticket = self.failManager.toBan()
-						self.jail.putFailTicket(ticket)
-				except FailManagerEmpty:
-					self.failManager.cleanup(MyTime.time())
+				logSys.error("Caught unhandled exception in main cycle: %r", e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+				# incr common error counter:
+				self.commonError()
 
+		# close journal:
+		try:
+			if self.__journal:
+				self.__journal.close()
+		except Exception as e: # pragma: no cover
+			logSys.error("Close journal failed: %r", e,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		logSys.debug((self.jail is not None and self.jail.name
                       or "jailless") +" filter terminated")
 		return True
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index e70eaddd..aa4a38cc 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -239,19 +239,31 @@ class Jail(object):
 		Once stated, also queries the persistent database to reinstate
 		any valid bans.
 		"""
+		logSys.debug("Starting jail %r", self.name)
 		self.filter.start()
 		self.actions.start()
 		self.restoreCurrentBans()
 		logSys.info("Jail %r started", self.name)
 
-	def stop(self):
+	def stop(self, stop=True, join=True):
 		"""Stop the jail, by stopping filter and actions threads.
 		"""
-		self.filter.stop()
-		self.actions.stop()
-		self.filter.join()
-		self.actions.join()
-		logSys.info("Jail '%s' stopped" % self.name)
+		if stop:
+			logSys.debug("Stopping jail %r", self.name)
+		for obj in (self.filter, self.actions):
+			try:
+				## signal to stop filter / actions:
+				if stop:
+					obj.stop()
+				## wait for end of threads:
+				if join:
+					if obj.isAlive():
+						obj.join()
+			except Exception as e:
+				logSys.error("Stop %r of jail %r failed: %s", obj, self.name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+		if join:
+			logSys.info("Jail %r stopped", self.name)
 
 	def isAlive(self):
 		"""Check jail "isAlive" by checking filter and actions threads.
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 97345331..a21b1fce 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -210,14 +210,14 @@ class Server:
 		if self.__db is not None:
 			self.__db.addJail(self.__jails[name])
 		
-	def delJail(self, name, stop=True):
+	def delJail(self, name, stop=True, join=True):
 		jail = self.__jails[name]
-		if stop and jail.isAlive():
-			logSys.debug("Stopping jail %r" % name)
-			jail.stop()
-		if self.__db is not None:
-			self.__db.delJail(jail)
-		del self.__jails[name]
+		if join or jail.isAlive():
+			jail.stop(stop=stop, join=join)
+		if join:
+			if self.__db is not None:
+				self.__db.delJail(jail)
+			del self.__jails[name]
 
 	def startJail(self, name):
 		with self.__lock:
@@ -237,8 +237,12 @@ class Server:
 	def stopAllJail(self):
 		logSys.info("Stopping all jails")
 		with self.__lock:
+			# 1st stop all jails (signal and stop actions/filter thread):
 			for name in self.__jails.keys():
-				self.delJail(name, stop=True)
+				self.delJail(name, stop=True, join=False)
+			# 2nd wait for end and delete jails:
+			for name in self.__jails.keys():
+				self.delJail(name, stop=False, join=True)
 
 	def reloadJails(self, name, opts, begin):
 		if begin:
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index a8a00257..74406363 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -52,8 +52,8 @@ class Utils():
 	"""Utilities provide diverse static methods like executes OS shell commands, etc.
 	"""
 
-	DEFAULT_SLEEP_TIME = 0.1
-	DEFAULT_SLEEP_INTERVAL = 0.01
+	DEFAULT_SLEEP_TIME = 2
+	DEFAULT_SLEEP_INTERVAL = 0.2
 
 
 	class Cache(object):

From c0373a715807457db92cf6e602ccc93580f6541f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 18:56:21 +0200
Subject: [PATCH 187/623] repair typo bug in reloading for systemd-filter;
 JailThread get method `join` for safe usage of it, also for not started
 threads (test-cases or in case of error), that will be used for cleanup resp.
 wait purposes also (see join of pyinotify-filter);

---
 fail2ban/server/filter.py          | 10 +++++++---
 fail2ban/server/filtergamin.py     |  2 +-
 fail2ban/server/filterpoll.py      |  4 +---
 fail2ban/server/filterpyinotify.py | 17 ++++++++++++-----
 fail2ban/server/filtersystemd.py   | 17 +++++++++++------
 fail2ban/server/jail.py            |  3 +--
 fail2ban/server/jailthread.py      | 15 +++++++++++++--
 fail2ban/server/transmitter.py     |  2 +-
 8 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index ac64fb86..13538c66 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -100,6 +100,10 @@ class Filter(JailThread):
 	def __repr__(self):
 		return "%s(%r)" % (self.__class__.__name__, self.jail)
 
+	@property
+	def jailName(self):
+		return (self.jail is not None and self.jail.name or "~jailless~")
+
 	def clearAllParams(self):
 		""" Clear all lists/dicts parameters (used by reloading)
 		"""
@@ -135,7 +139,7 @@ class Filter(JailThread):
 			if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
 				logSys.warning(
 					"Mutliline regex set for jail %r "
-					"but maxlines not greater than 1", self.jail.name)
+					"but maxlines not greater than 1", self.jailName)
 		except RegexException as e:
 			logSys.error(e)
 			raise e
@@ -420,7 +424,7 @@ class Filter(JailThread):
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
 		if log_ignore:
-			logSys.info("[%s] Ignore %s by %s" % (self.jail.name, ip, ignore_source))
+			logSys.info("[%s] Ignore %s by %s" % (self.jailName, ip, ignore_source))
 
 	def getIgnoreIP(self):
 		return self.__ignoreIpList
@@ -491,7 +495,7 @@ class Filter(JailThread):
 				if self.inIgnoreIPList(ip, log_ignore=True):
 					continue
 				logSys.info(
-					"[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
+					"[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 				)
 				tick = FailTicket(ip, unixTime, lines, data=fail)
 				self.failManager.addFailure(tick)
diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index a8d03ebd..106e4c0f 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -132,7 +132,7 @@ class FilterGamin(FileFilter):
 			Utils.wait_for(lambda: not self.active or self._handleEvents(),
 				self.sleeptime)
 			self.ticks += 1
-		logSys.debug(self.jail.name + ": filter terminated")
+		logSys.debug("[%s] filter terminated", self.jailName)
 		return True
 
 	def stop(self):
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index d0660f15..abf16e1b 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -123,9 +123,7 @@ class FilterPoll(FileFilter):
 				except FailManagerEmpty:
 					self.failManager.cleanup(MyTime.time())
 				self.__modified = False
-		logSys.debug(
-			(self.jail is not None and self.jail.name or "jailless") +
-					 " filter terminated")
+		logSys.debug("[%s] filter terminated", self.jailName)
 		return True
 
 	##
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 9b444844..6ddf1691 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -198,7 +198,7 @@ class FilterPyinotify(FileFilter):
 			prcevent, timeout=self.sleeptime)
 		self.__notifier.check_events = self.__check_events
 		self.__notifier.start()
-		logSys.debug("pyinotifier started for %s.", self.jail.name)
+		logSys.debug("[%s] filter started (pyinotifier)", self.jailName)
 		return True
 
 	##
@@ -206,15 +206,22 @@ class FilterPyinotify(FileFilter):
 
 	def stop(self):
 		super(FilterPyinotify, self).stop()
-
 		# Stop the notifier thread
 		self.__notifier.stop()
-		self.__notifier.join()			# to not exit before notifier does
-		self.__cleanup()				# for pedantic ones
+
+	##
+	# Wait for exit with cleanup.
+
+	def join(self):
+		self.__cleanup()
+		super(FilterPyinotify, self).join()
+		logSys.debug("[%s] filter terminated (pyinotifier)", self.jailName)
 
 	##
 	# Deallocates the resources used by pyinotify.
 
 	def __cleanup(self):
-		self.__notifier = None
+		if self.__notifier:
+			self.__notifier.join()			# to not exit before notifier does
+			self.__notifier = None
 		self.__monitor = None
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index a29085ba..310d15b3 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -130,7 +130,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			self.resetJournalMatches()
 			raise
 		else:
-			logSys.info("Added journal match for: %r", " ".join(match))
+			logSys.info("[%s] Added journal match for: %r", self.jailName, 
+				" ".join(match))
 	##
 	# Reset a journal match filter called on removal or failure
 	#
@@ -138,7 +139,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 	def resetJournalMatches(self):
 		self.__journal.flush_matches()
-		logSys.debug("Flushed all journal matches")
+		logSys.debug("[%s] Flushed all journal matches", self.jailName)
 		match_copy = self.__matches[:]
 		self.__matches = []
 		try:
@@ -157,14 +158,17 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def delJournalMatch(self, match=None):
 		# clear all:
 		if match is None:
+			if not self.__matches:
+				return
 			del self.__matches[:]
 		# delete by index:
 		elif match in self.__matches:
 			del self.__matches[self.__matches.index(match)]
 		else:
-			raise ValueError("Match not found")
+			raise ValueError("Match %r not found" % match)
 		self.resetJournalMatches()
-		logSys.info("Removed journal match for: %r" % " ".join(match))
+		logSys.info("[%s] Removed journal match for: %r", self.jailName, 
+			match if match else '*')
 
 	##
 	# Get current journal match filter
@@ -214,8 +218,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP',
 				logentry.get('__REALTIME_TIMESTAMP'))
-		logSys.debug("Read systemd journal entry: %r" %
-			"".join([date.isoformat(), logline]))
+		logSys.debug("[%s] Read systemd journal entry: %s %s", self.jailName,
+			date.isoformat(), logline)
 		## use the same type for 1st argument:
 		return ((logline[:0], date.isoformat(), logline),
 			time.mktime(date.timetuple()) + date.microsecond/1.0E6)
@@ -302,6 +306,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				# incr common error counter:
 				self.commonError()
 
+		logSys.debug("[%s] filter terminated", self.jailName)
 		# close journal:
 		try:
 			if self.__journal:
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index aa4a38cc..39fdd959 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -257,8 +257,7 @@ class Jail(object):
 					obj.stop()
 				## wait for end of threads:
 				if join:
-					if obj.isAlive():
-						obj.join()
+					obj.join()
 			except Exception as e:
 				logSys.error("Stop %r of jail %r failed: %s", obj, self.name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py
index 39a86c2b..280b2171 100644
--- a/fail2ban/server/jailthread.py
+++ b/fail2ban/server/jailthread.py
@@ -53,8 +53,8 @@ class JailThread(Thread):
 		super(JailThread, self).__init__(name=name)
 		## Should going with main thread also:
 		self.daemon = True
-		## Control the state of the thread.
-		self.active = False
+		## Control the state of the thread (None - was not started, True - active, False - stopped).
+		self.active = None
 		## Control the idle state of the thread.
 		self.idle = False
 		## The time the thread sleeps in the loop.
@@ -93,3 +93,14 @@ class JailThread(Thread):
 		"""Abstract - Called when thread starts, thread stops when returns.
 		"""
 		pass
+
+	def join(self):
+		""" Safer join, that could be called also for not started (or ended) threads (used for cleanup).
+		"""
+		## if cleanup needed - create derivate and call it before join...
+
+		## if was really started - should call join:
+		if self.active is not None:
+			super(JailThread, self).join()
+
+
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 19cf51ef..2f5be043 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -102,7 +102,7 @@ class Transmitter:
 					self.__commandHandler(cmd)
 			finally:
 				self.__server.reloadJails(*opts, begin=False)
-			return None
+			return 'OK'
 		elif len(command) >= 2 and command[0] == "unban":
 			# unban in all jails:
 			value = command[1:]

From 48ebe3e73594ad43de7246eff36a509e6118e34b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 20:15:12 +0200
Subject: [PATCH 188/623] FilterPyinotify: high cpu load fix - timeout for
 pyinotify must be set in milliseconds (our time values are floats contain
 seconds);

---
 fail2ban/server/filterpyinotify.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 6ddf1691..56d6ef83 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -194,8 +194,9 @@ class FilterPyinotify(FileFilter):
 	def run(self):
 		prcevent = pyinotify.ProcessEvent()
 		prcevent.process_default = self.__process_default
+		## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds)
 		self.__notifier = pyinotify.ThreadedNotifier(self.__monitor,
-			prcevent, timeout=self.sleeptime)
+			prcevent, timeout=self.sleeptime * 1000)
 		self.__notifier.check_events = self.__check_events
 		self.__notifier.start()
 		logSys.debug("[%s] filter started (pyinotifier)", self.jailName)

From ab0c28260bc91532bd576fd6bcb05301976e711a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 20:16:28 +0200
Subject: [PATCH 189/623] switch down log level for some annoying messages to
 tracedebug or heavydebug (to 7 or even 5); added verification of specified
 log-level before transmitting to the server; numeric log-level allowed now in
 server (resp. fail2ban.conf);

---
 fail2ban/__init__.py               |  5 ++++-
 fail2ban/client/fail2banreader.py  |  4 +++-
 fail2ban/helpers.py                | 10 ++++++++++
 fail2ban/protocol.py               |  3 ++-
 fail2ban/server/filterpoll.py      |  4 ++--
 fail2ban/server/filterpyinotify.py |  2 +-
 fail2ban/server/filtersystemd.py   |  2 +-
 fail2ban/server/server.py          | 15 ++++++---------
 fail2ban/tests/servertestcase.py   |  2 ++
 man/fail2ban-client.1              |  4 +++-
 man/fail2ban-regex.1               |  5 +++++
 man/jail.conf.5                    |  2 +-
 12 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/fail2ban/__init__.py b/fail2ban/__init__.py
index cd92dbab..0f0fc3ec 100644
--- a/fail2ban/__init__.py
+++ b/fail2ban/__init__.py
@@ -26,8 +26,11 @@ __license__ = "GPL"
 
 import logging.handlers
 
-# Custom debug level
+# Custom debug levels
+logging.TRACEDEBUG = 7
 logging.HEAVYDEBUG = 5
+logging.addLevelName(logging.TRACEDEBUG, 'TRACE')
+logging.addLevelName(logging.HEAVYDEBUG, 'HEAVY')
 
 """
 Below derived from:
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index 8e1b0e5c..c81d585e 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 from .configreader import ConfigReader
-from ..helpers import getLogger
+from ..helpers import getLogger, str2LogLevel
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -58,6 +58,8 @@ class Fail2banReader(ConfigReader):
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
 		if updateMainOpt:
 			self.__opts.update(updateMainOpt)
+		# check given log-level:
+		str2LogLevel(self.__opts.get('loglevel', 0))
 	
 	def convert(self):
 		# Ensure logtarget/level set first so any db errors are captured
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 2738f5e6..bbed61cf 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -129,6 +129,16 @@ def getLogger(name):
 		name = "fail2ban.%s" % name.rpartition(".")[-1]
 	return logging.getLogger(name)
 
+def str2LogLevel(value):
+	try:
+		if isinstance(value, int) or value.isdigit():
+			ll = int(value)
+		else:
+			ll = getattr(logging, value)
+	except AttributeError:
+		raise ValueError("Invalid log level %r" % value)
+	return ll
+
 
 def excepthook(exctype, value, traceback):
 	"""Except hook used to log unhandled exceptions to Fail2Ban log
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 879183e5..d1c33d88 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -61,7 +61,8 @@ protocol = [
 ["help", "return this output"], 
 ["version", "return the server version"],
 ['', "LOGGING", ""],
-["set loglevel <LEVEL>", "sets logging level to <LEVEL>. Levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG"], 
+["set loglevel <LEVEL>", "sets logging level to <LEVEL>. Levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, "
+	"DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5)"], 
 ["get loglevel", "gets the logging level"], 
 ["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG or a file"], 
 ["get logtarget", "gets logging target"], 
diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index abf16e1b..59310bed 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -137,10 +137,10 @@ class FilterPoll(FileFilter):
 			logStats = os.stat(filename)
 			stats = logStats.st_mtime, logStats.st_ino, logStats.st_size
 			pstats = self.__prevStats.get(filename, (0))
-			if logSys.getEffectiveLevel() <= 7:
+			if logSys.getEffectiveLevel() <= 5:
 				# we do not want to waste time on strftime etc if not necessary
 				dt = logStats.st_mtime - pstats[0]
-				logSys.log(7, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s",
+				logSys.log(5, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s",
 				           filename, pstats, stats, dt)
 				# os.system("stat %s | grep Modify" % filename)
 			self.__file404Cnt[filename] = 0
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 56d6ef83..73c82099 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -76,7 +76,7 @@ class FilterPyinotify(FileFilter):
 		logSys.debug("Created FilterPyinotify")
 
 	def callback(self, event, origin=''):
-		logSys.debug("%sCallback for Event: %s", origin, event)
+		logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event)
 		path = event.pathname
 		if event.mask & ( pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO ):
 			# skip directories altogether
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 310d15b3..0d720a5e 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -218,7 +218,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP',
 				logentry.get('__REALTIME_TIMESTAMP'))
-		logSys.debug("[%s] Read systemd journal entry: %s %s", self.jailName,
+		logSys.log(5, "[%s] Read systemd journal entry: %s %s", self.jailName,
 			date.isoformat(), logline)
 		## use the same type for 1st argument:
 		return ((logline[:0], date.isoformat(), logline),
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index a21b1fce..2ae96e15 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -38,7 +38,7 @@ from .filter import FileFilter, JournalFilter
 from .transmitter import Transmitter
 from .asyncserver import AsyncServer, AsyncServerException
 from .. import version
-from ..helpers import getLogger, excepthook
+from ..helpers import getLogger, str2LogLevel, excepthook
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -510,14 +510,11 @@ class Server:
 		with self.__loggingLock:
 			if self.__logLevel == value:
 				return
-			try:
-				ll = getattr(logging, value)
-				# don't change real log-level if running from the test cases:
-				getLogger("fail2ban").setLevel(
-					ll if DEF_LOGTARGET != "INHERITED" or ll < logging.DEBUG else DEF_LOGLEVEL)
-				self.__logLevel = value
-			except AttributeError:
-				raise ValueError("Invalid log level %r" % value)
+			ll = str2LogLevel(value)
+			# don't change real log-level if running from the test cases:
+			getLogger("fail2ban").setLevel(
+				ll if DEF_LOGTARGET != "INHERITED" or ll < logging.DEBUG else DEF_LOGLEVEL)
+			self.__logLevel = value
 	
 	##
 	# Get the logging level.
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d044da4b..ebc39d9b 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -842,6 +842,8 @@ class TransmitterLogging(TransmitterBase):
 
 	def testLogLevel(self):
 		self.setGetTest("loglevel", "HEAVYDEBUG")
+		self.setGetTest("loglevel", "TRACEDEBUG")
+		self.setGetTest("loglevel", "9")
 		self.setGetTest("loglevel", "DEBUG")
 		self.setGetTest("loglevel", "INFO")
 		self.setGetTest("loglevel", "NOTICE")
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 33bce652..281d8f5b 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -120,7 +120,9 @@ LOGGING
 \fBset loglevel <LEVEL>\fR
 sets logging level to <LEVEL>.
 Levels: CRITICAL, ERROR, WARNING,
-NOTICE, INFO, DEBUG
+NOTICE, INFO, DEBUG, TRACEDEBUG,
+HEAVYDEBUG or corresponding
+numeric value (50\-5)
 .TP
 \fBget loglevel\fR
 gets the logging level
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 342d18f7..44e13c86 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -51,6 +51,11 @@ File encoding. Default: system locale
 \fB\-r\fR, \fB\-\-raw\fR
 Raw hosts, don't resolve dns
 .TP
+\fB\-\-usedns\fR=\fI\,USEDNS\/\fR
+DNS specified replacement of tags <HOST> in regexp
+('yes' \- matches all form of hosts, 'no' \- IP
+addresses only)
+.TP
 \fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fI\,MAXLINES\/\fR
 maxlines for multi\-line regex
 .TP
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 865c689e..51a00cdc 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -127,7 +127,7 @@ These files have one section, [Definition].
 The items that can be set are:
 .TP
 .B loglevel
-verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG. Default: ERROR
+verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40)
 .TP
 .B logtarget
 log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR

From e7fa74b98911abc52d9f2b6ee54e6dec0f6085ac Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 21:32:39 +0200
Subject: [PATCH 190/623] smaller inertance inside test-cases (amend to
 d153555a07ec81dabe4678ea06c8e644afc8ec40 with decreasing default wait
 operation that litle bit speedup test-cases)

---
 fail2ban/tests/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 76bb724c..91350b1d 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -243,6 +243,9 @@ def initTests(opts):
 			raise unittest.SkipTest('Skip test because of "--fast"')
 		unittest.F2B.SkipIfFast = F2B_SkipIfFast
 	else:
+		# smaller inertance inside test-cases (litle speedup):
+		Utils.DEFAULT_SLEEP_TIME = 0.25
+		Utils.DEFAULT_SLEEP_INTERVAL = 0.025
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):

From e00be5f308c98a53bde277baa24d20b6712b9644 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 22 Sep 2016 22:09:08 +0200
Subject: [PATCH 191/623] Fixed sporadically error in testCymruInfoNxdomain,
 because of unsorted values: ``` AssertionError: Dictionaries differ:
 {'country': ['unknown', 'nxdomain'], 'asn': ['4565', 'nxdomain'], 'rir':
 ['other', 'nxdomain']} != {'country': ['nxdomain', 'unknown'], 'asn':
 ['nxdomain', '4565'], 'rir': ['nxdomain', 'other']} ``` Added assertDictEqual
 for compatibility to early python versions (< 2.7);

---
 fail2ban/tests/banmanagertestcase.py | 13 ++++++-------
 fail2ban/tests/misctestcase.py       |  4 ++++
 fail2ban/tests/utils.py              | 20 ++++++++++++++------
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 4a86102c..de128b70 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -28,7 +28,6 @@ import unittest
 
 from ..server.banmanager import BanManager
 from ..server.ticket import BanTicket
-from .utils import assert_dict_equal
 
 class AddFailure(unittest.TestCase):
 	def setUp(self):
@@ -131,7 +130,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 
 	def testCymruInfo(self):
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
-		assert_dict_equal(cymru_info,
+		self.assertDictEqual(cymru_info,
 						  {"asn": [self.__asn],
 						   "country": [self.__country],
 						   "rir": [self.__rir]})
@@ -158,7 +157,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		ticket = BanTicket("0.0.0.0", 1167605999.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
-		assert_dict_equal(cymru_info,
+		self.assertDictEqual(cymru_info,
 						  {"asn": ["nxdomain"],
 						   "country": ["nxdomain"],
 						   "rir": ["nxdomain"]})
@@ -169,7 +168,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		ticket = BanTicket("10.0.0.0", 1167606000.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
-		assert_dict_equal(cymru_info,
-						  {"asn": ["nxdomain", "4565",],
-						   "country": ["nxdomain", "unknown"],
-						   "rir": ["nxdomain", "other"]})
+		self.assertDictEqual(dict((k, sorted(v)) for k, v in cymru_info.iteritems()),
+						  {"asn": sorted(["nxdomain", "4565",]),
+						   "country": sorted(["nxdomain", "unknown"]),
+						   "rir": sorted(["nxdomain", "other"])})
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 8fb393ea..450904d5 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -284,6 +284,10 @@ class TestsUtilsTest(LogCaptureTestCase):
 			self.assertLogged, 'test_zyx', 'zyx', all=False)
 		self._testAssertionErrorRE(r"All of the .* were found present in the log",
 			self.assertNotLogged, 'test', 'xyz', all=False)
+		## assertDictEqual:
+		self.assertDictEqual({'A': [1, 2]}, {'A': [1, 2]})
+		self.assertRaises(AssertionError, self.assertDictEqual, 
+			{'A': [1, 2]}, {'A': [2, 1]})
 
 	def testFormatterWithTraceBack(self):
 		strout = StringIO()
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 91350b1d..eecb94b8 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -466,6 +466,20 @@ def gatherTests(regexps=None, opts=None):
 # Forwards compatibility of unittest.TestCase for some early python versions
 #
 
+if not hasattr(unittest.TestCase, 'assertDictEqual'):
+	import difflib, pprint
+	def assertDictEqual(self, d1, d2, msg=None):
+		self.assert_(isinstance(d1, dict), 'First argument is not a dictionary')
+		self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary')
+		if d1 != d2:
+			standardMsg = '%r != %r' % (d1, d2)
+			diff = ('\n' + '\n'.join(difflib.ndiff(
+				pprint.pformat(d1).splitlines(),
+				pprint.pformat(d2).splitlines())))
+			msg = msg or (standardMsg + diff)
+			self.fail(msg)
+	unittest.TestCase.assertDictEqual = assertDictEqual
+
 if not hasattr(unittest.TestCase, 'assertRaisesRegexp'):
 	def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs):
 		try:
@@ -673,9 +687,3 @@ class LogCaptureTestCase(unittest.TestCase):
 
 
 pid_exists = Utils.pid_exists
-
-# Python 2.6 compatibility. in 2.7 assertDictEqual
-def assert_dict_equal(a, b):
-	assert isinstance(a, dict), "Object is not dictionary: %r" % a
-	assert isinstance(b, dict), "Object is not dictionary: %r" % b
-	assert a==b, "Dictionaries differ:\n%r !=\n%r" % (a, b)

From 004879b5b15c8d404a22935bfd0c41579faf3d87 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 23 Sep 2016 09:32:10 +0200
Subject: [PATCH 192/623] code review: switch MAX_TIME to 0X7FFFFFFFFFFF (is
 enough, because 4461763-th year, but better performance)

---
 fail2ban/server/banmanager.py | 4 ++--
 fail2ban/server/ticket.py     | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 6da5366c..9aa69f5a 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -57,7 +57,7 @@ class BanManager:
 		## Total number of banned IP address
 		self.__banTotal = 0
 		## The time for next unban process (for performance and load reasons):
-		self.__nextUnbanTime = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
+		self.__nextUnbanTime = BanTicket.MAX_TIME
 	
 	##
 	# Set the ban time.
@@ -322,7 +322,7 @@ class BanManager:
 
 			# Gets the list of ticket to remove (thereby correct next unban time).
 			unBanList = {}
-			self.__nextUnbanTime = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
+			self.__nextUnbanTime = BanTicket.MAX_TIME
 			for fid,ticket in self.__banList.iteritems():
 				# current time greater as end of ban - timed out:
 				eob = ticket.getEndOfBanTime(self.__banTime)
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index 2f9a88ed..c7bb4d47 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -35,6 +35,8 @@ logSys = getLogger(__name__)
 
 
 class Ticket(object):
+
+	MAX_TIME = 0X7FFFFFFFFFFF ;# 4461763-th year
 	
 	RESTORED = 0x01
 	BANNED   = 0x08
@@ -112,7 +114,7 @@ class Ticket(object):
 		bantime = (self._banTime if self._banTime is not None else defaultBT)
 		# permanent
 		if bantime == -1:
-			return 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL
+			return Ticket.MAX_TIME
 		# unban time (end of ban):
 		return self._time + bantime
 

From 449c46aec45a39e6406894f77ee965a6322bcb1e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 23 Sep 2016 15:21:23 +0200
Subject: [PATCH 193/623] extended test-cases (coverage)

---
 fail2ban/tests/banmanagertestcase.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index de128b70..b627599d 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -95,9 +95,28 @@ class AddFailure(unittest.TestCase):
 
 	def testUnban(self):
 		btime = self.__banManager.getBanTime()
+		stime = self.__ticket.getTime()
 		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
 		self.assertTrue(self.__banManager._inBanList(self.__ticket))
-		self.assertEqual(self.__banManager.unBanList(self.__ticket.getTime() + btime + 1), [self.__ticket])
+		self.assertEqual(self.__banManager.unBanList(stime), [])
+		self.assertEqual(self.__banManager.unBanList(stime + btime + 1), [self.__ticket])
+		self.assertEqual(self.__banManager.size(), 0)
+		## again, but now we will prolong ban-time and then try to unban again (1st too early):
+		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
+		# prolong ban:
+		ticket = BanTicket(self.__ticket.getID(), stime + 600)
+		self.assertFalse(self.__banManager.addBanTicket(ticket))
+		# try unban too early:
+		self.assertEqual(len(self.__banManager.unBanList(stime + btime + 1)), 0)
+		# try unban using correct time:
+		self.assertEqual(len(self.__banManager.unBanList(stime + btime + 600 + 1)), 1)
+		## again, but now we test removing tickets particular (to test < 2/3-rule):
+		for i in range(5):
+			ticket = BanTicket('193.168.0.%s' % i, stime)
+			ticket.setBanTime(ticket.getBanTime(btime) + i*10)
+			self.assertTrue(self.__banManager.addBanTicket(ticket))
+		self.assertEqual(len(self.__banManager.unBanList(stime + btime + 1*10 + 1)), 2)
+		self.assertEqual(len(self.__banManager.unBanList(stime + btime + 5*10 + 1)), 3)
 		self.assertEqual(self.__banManager.size(), 0)
 
 	def testUnbanPermanent(self):

From 5e4fdb60c8a415dcf0c702a8ee2896cd25d5550d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 26 Sep 2016 10:50:02 +0200
Subject: [PATCH 194/623] extended test-cases (coverage)

---
 fail2ban/server/database.py        | 2 +-
 fail2ban/tests/databasetestcase.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 120619f9..77786f57 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -211,7 +211,7 @@ class Fail2BanDb(object):
 				if newversion == Fail2BanDb.__version__:
 					logSys.warning( "Database updated from '%i' to '%i'",
 						version, newversion)
-				else:
+				else: # pragma: no cover
 					logSys.error( "Database update failed to achieve version '%i'"
 						": updated from '%i' to '%i'",
 						Fail2BanDb.__version__, version, newversion)
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 02d2c79b..b1c68a8f 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -353,6 +353,11 @@ class DatabaseTest(LogCaptureTestCase):
 		# be returned
 		tickets = self.db.getBansMerged(bantime=-1)
 		self.assertEqual(len(tickets), 2)
+		# getCurrentBans:
+		tickets = self.db.getCurrentBans(jail=self.jail)
+		self.assertEqual(len(tickets), 2)
+		ticket = self.db.getCurrentBans(jail=None, ip="127.0.0.1");
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
 
 	def testActionWithDB(self):
 		# test action together with database functionality

From 5151c4fa6d36dd0e1569a2a8fbb0a797d3f7bb9a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 26 Sep 2016 11:15:59 +0200
Subject: [PATCH 195/623] ChangeLog entries added

---
 ChangeLog | 85 +++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 73 insertions(+), 12 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 204c9559..0c1c3676 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -12,14 +12,22 @@ ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
 TODO: implementing of options resp. other tasks from PR #1346
 
 ### Fixes
-* [grave] memory leak's fixed (gh-1277, gh-1234)
-* tricky bug fix: last position of log file will be never retrieved (gh-795),
+* [Grave] memory leak's fixed (gh-1277, gh-1234)
+* Tricky bug fix: last position of log file will be never retrieved (gh-795),
   because of CASCADE all log entries will be deleted from logs table together with jail, 
   if used "INSERT OR REPLACE" statement
-* asyncserver (asyncore) code fixed and test cases repaired (again gh-161)
+* Asyncserver (asyncore) code fixed and test cases repaired (again gh-161)
 * testSocket: sporadical bug repaired - wait for server thread starts a socket (listener)
 * testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash,
   kill tree in any case (gh-1155)
+* Fixed high-load of pyinotify-backend,
+  see https://github.com/fail2ban/fail2ban/issues/885#issuecomment-248964591
+* Database: stability fix - repack cursor iterator as long as locked
+* File filter backends: stability fix for sporadically errors - always close file
+  handle, otherwise may be locked (prevent log-rotate, etc.)
+* Pyinotify-backend: stability fix for sporadically errors in multi-threaded
+  environment (without lock)
+* Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values
 
 ### New Features
 * IPv6 support:
@@ -33,18 +41,33 @@ TODO: implementing of options resp. other tasks from PR #1346
     - new conditional section functionality used in config resp. includes:
       - [Init?family=inet4] - IPv4 qualified hosts only
       - [Init?family=inet6] - IPv6 qualified hosts only
+* New reload functionality (now totally without restart, unbanning/rebanning, etc.),
+  see gh-1557
+* Several commands extended and new commands introduced:
+  - `restart [--unban] [--if-exists] <JAIL>` - restarts the jail \<JAIL\>
+    (alias for `reload --restart ... <JAIL>`)
+  - `reload [--restart] [--unban] [--all]` - reloads the configuration without restarting
+    of the server, the option `--restart` activates completely restarting of affected jails,
+    thereby can unban IP addresses (if option `--unban` specified)
+  - `reload [--restart] [--unban] [--if-exists] <JAIL>` - reloads the jail \<JAIL\>,
+    or restarts it (if option `--restart` specified), at the same time unbans all IP addresses
+    banned in this jail, if option `--unban` specified
+  - `unban --all` - unbans all IP addresses (in all jails and database)
+  - `unban <IP> ... <IP>` - unbans \<IP\> (in all jails and database) (see gh-1388)
+* New command action parameter `actionrepair` - command executed in order to restore
+  sane environment in error case of `actioncheck`.
 
 ### Enhancements
-* huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)
-* datedetector: in-place reordering using hits and last used time: 
+* Huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)
+* Datedetector: in-place reordering using hits and last used time: 
   matchTime, template list etc. rewritten because of performance degradation
-* prevent out of memory situation if many IP's makes extremely many failures (maxEntries)
-* introduced string to seconds (str2seconds) for configuration entries with time,
+* Prevent out of memory situation if many IP's makes extremely many failures (maxEntries)
+* Introduced string to seconds (str2seconds) for configuration entries with time,
   use `1h` instead of `3600`, `1d` instead of `86400`, etc
 * seekToTime - prevent completely read of big files first time (after start of service), 
   initial seek to start time using half-interval search algorithm (see issue gh-795)
-* ticket and some other modules prepared to easy merge with newest version of 'ban-time-incr'
-* cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name,
+* Ticket and some other modules prepared to easy merge with newest version of 'ban-time-incr'
+* Cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name,
   especially for wrong dns or lazy dns-system
 * FailManager memory-optimization: increases performance, 
   prevents memory leakage, because don't copy failures list on some operations
@@ -54,14 +77,52 @@ TODO: implementing of options resp. other tasks from PR #1346
     - `-g`, `--no-gamin` to prevent running of tests that require the gamin (slow)
     - `-m`, `--memory-db` - run database tests using memory instead of file
     - `-i`, `--ignore` - negate [regexps] filter to ignore tests matched specified regexps
-* background servicing: prevents memory leak on some platforms/python versions, using forced GC
+* Background servicing: prevents memory leak on some platforms/python versions, using forced GC
   in periodic intervals (latency and threshold)
 * executeCmd partially moved from action to new module utils
-* several functionality of class `DNSUtils` moved to new class `IPAddr`, 
+* Several functionality of class `DNSUtils` moved to new class `IPAddr`, 
   both classes moved to new module `ipdns`
-* pseudo-conditional section introduced, for conditional substitution resp. 
+* Pseudo-conditional section introduced, for conditional substitution resp. 
   evaluation of parameters for different family qualified hosts, 
   syntax `[Section?family=inet6]` (currently use for IPv6-support only).
+* All the backends were rewritten to get reload-possibility, performance increased,
+  so fewer greedy regarding cpu- resp. system-load now
+* Numeric log-level allowed now in server (resp. fail2ban.conf);
+* Implemented better error handling in some multi-threaded routines; shutdown of jails
+  rewritten (faster and safer, does not breaks shutdown process if some error occurred)
+* Possibility for overwriting some configuration options (read with config-readers)
+  with command line option, e. g.:
+```bash
+## start server with DEBUG log-level (ignore level read from fail2ban.conf):
+fail2ban-client --loglevel DEBUG start
+## or
+fail2ban-server -c /cfg/path --loglevel DEBUG start
+## keep server log-level by reload (without restart it)
+fail2ban-client --loglevel DEBUG reload
+## switch log-level back to INFO:
+fail2ban-client set loglevel INFO
+```
+* Optimized BanManager: increase performance, fewer system load, try to prevent
+  memory leakage:
+  - better ban/unban handling within actions (e.g. used dict instead of list)
+  - don't copy bans resp. its list on some operations;
+  - added new unbantime handling to relieve unBanList (prevent permanent
+    searching for tickets to unban)
+  - prefer failure-ID as identifier of the ticket to its IP (most of the time
+    the same, but it can be something else e.g. user name in some complex jails,
+    as introduced in 0.10)
+* Regexp enhancements:
+  - build replacement of `<HOST>` substitution corresponding parameter
+    `usedns` - dns-part will be added only if `usedns` is not `no`,
+    also using fail2ban-regex
+  - new replacement for `<ADDR>` in opposition to `<HOST>`, for separate
+    usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6`
+    together, without host (dns)
+* fail2ban-testcases:
+  - `assertLogged` extended with parameter wait (to wait up to specified timeout,
+    before we throw assert exception) + test cases rewritten using that
+  - added `assertDictEqual` for compatibility to early python versions (< 2.7);
+  - new `with_foreground_server_thread` decorator to test several client/server commands
 
 
 ver. 0.9.6 (2016/XX/XX) - wanna-be-released

From b011cf17b26960a6f9c2854a259a3c996833df77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 29 Sep 2016 20:47:10 +0200
Subject: [PATCH 196/623] increase performance of executeCmd (actions), thereby
 introduced new shorter interval for fast operations (leaves unchanged default
 wait operation intervals (sleep time, threshold interval) - for the same
 inertance, to save same system (load by many jails resp. log files); extends
 wait_for with callable timeout (test case fixed);

---
 fail2ban/server/utils.py         | 54 +++++++++++++++++++++++---------
 fail2ban/tests/actiontestcase.py |  4 +--
 fail2ban/tests/utils.py          |  2 ++
 3 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 74406363..7c13631e 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -54,6 +54,7 @@ class Utils():
 
 	DEFAULT_SLEEP_TIME = 2
 	DEFAULT_SLEEP_INTERVAL = 0.2
+	DEFAULT_SHORT_INTERVAL = 0.001
 
 
 	class Cache(object):
@@ -134,21 +135,22 @@ class Utils():
 		"""
 		stdout = stderr = None
 		retcode = None
-		if not callable(timeout):
-			stime = time.time()
-			timeout_expr = lambda: time.time() - stime <= timeout
-		else:
-			timeout_expr = timeout
 		popen = None
 		try:
 			popen = subprocess.Popen(
 				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell,
 				preexec_fn=os.setsid  # so that killpg does not kill our process
 			)
+			# wait with timeout for process has terminated:
 			retcode = popen.poll()
-			while retcode is None and timeout_expr():
-				time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
-				retcode = popen.poll()
+			if retcode is None:
+				def _popen_wait_end():
+					retcode = popen.poll()
+					return (True, retcode) if retcode is not None else None
+				retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL)
+				if retcode:
+					retcode = retcode[1]
+			# if timeout:
 			if retcode is None:
 				logSys.error("%s -- timed out after %s seconds." %
 					(realCmd, timeout))
@@ -202,18 +204,18 @@ class Utils():
 
 		success = False
 		if retcode == 0:
-			logSys.debug("%s -- returned successfully", realCmd)
+			logSys.debug("%-.40s -- returned successfully", realCmd)
 			success = True
 		elif retcode is None:
-			logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid))
+			logSys.error("%-.40s -- unable to kill PID %i", realCmd, popen.pid)
 		elif retcode < 0 or retcode > 128:
 			# dash would return negative while bash 128 + n
 			sigcode = -retcode if retcode < 0 else retcode - 128
-			logSys.error("%s -- killed with %s (return code: %s)" %
-				(realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode))
+			logSys.error("%-.40s -- killed with %s (return code: %s)",
+				realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)
 		else:
 			msg = _RETCODE_HINTS.get(retcode, None)
-			logSys.error("%s -- returned %i" % (realCmd, retcode))
+			logSys.error("%-.40s -- returned %i", realCmd, retcode)
 			if msg:
 				logSys.info("HINT on %i: %s", retcode, msg % locals())
 		return success if not output else (success, stdout, stderr, retcode)
@@ -221,7 +223,25 @@ class Utils():
 	@staticmethod
 	def wait_for(cond, timeout, interval=None):
 		"""Wait until condition expression `cond` is True, up to `timeout` sec
+
+		Parameters
+		----------
+		cond : callable
+			The expression to check condition 
+			(should return equivalent to bool True if wait successful).
+		timeout : float or callable
+			The time out for end of wait
+			(in seconds or callable that returns True if timeout occurred).
+		interval : float (optional)
+			Polling start interval for wait cycle in seconds.
+
+		Returns
+		-------
+		variable
+			The return value of the last call of `cond`, 
+			logical False (or None, 0, etc) if timeout occurred.
 		"""
+		#logSys.log(5, "  wait for %r, tout: %r / %r", cond, timeout, interval)
 		ini = 1  # to delay initializations until/when necessary
 		while True:
 			ret = cond()
@@ -229,10 +249,14 @@ class Utils():
 				return ret
 			if ini:
 				ini = stm = 0
-				time0 = time.time() + timeout
+				if not callable(timeout):
+					time0 = time.time() + timeout
+					timeout_expr = lambda: time.time() > time0
+				else:
+					timeout_expr = timeout
 				if not interval:
 					interval = Utils.DEFAULT_SLEEP_INTERVAL
-			if time.time() > time0:
+			if timeout_expr():
 				break
 			stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME)
 			time.sleep(stm)
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index cacb5e17..c676b346 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -353,8 +353,8 @@ class CommandActionTest(LogCaptureTestCase):
 		# timeout as long as pid-file was not created, but max 5 seconds
 		def getnasty_tout():
 			return (
-				getnastypid() is None
-				and time.time() - stime <= 5
+				getnastypid() is not None
+				or time.time() - stime > 5
 			)
 
 		def getnastypid():
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index eecb94b8..3c044951 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -239,6 +239,7 @@ def initTests(opts):
 		# (prevent long sleeping during test cases ... less time goes to sleep):
 		Utils.DEFAULT_SLEEP_TIME = 0.0025
 		Utils.DEFAULT_SLEEP_INTERVAL = 0.0005
+		Utils.DEFAULT_SHORT_INTERVAL = 0.0001
 		def F2B_SkipIfFast():
 			raise unittest.SkipTest('Skip test because of "--fast"')
 		unittest.F2B.SkipIfFast = F2B_SkipIfFast
@@ -246,6 +247,7 @@ def initTests(opts):
 		# smaller inertance inside test-cases (litle speedup):
 		Utils.DEFAULT_SLEEP_TIME = 0.25
 		Utils.DEFAULT_SLEEP_INTERVAL = 0.025
+		Utils.DEFAULT_SHORT_INTERVAL = 0.0005
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):

From b615ba49ff3047b9b6e2e50e78e8db5eaf3e050f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 29 Sep 2016 20:48:33 +0200
Subject: [PATCH 197/623] disengage testExecuteTimeout test-case from -fast
 option, just make it faster (timeout shorter) in this case

---
 fail2ban/tests/actiontestcase.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index c676b346..403f7ea6 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -324,16 +324,13 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertLogged('HINT on 127: "Command not found"')
 
 	def testExecuteTimeout(self):
-		unittest.F2B.SkipIfFast()
 		stime = time.time()
-		# Should take a minute
-		self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=1))
+		timeout = 1 if not unittest.F2B.fast else 0.01
+		# Should take a 30 seconds (so timeout will occur)
+		self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=timeout))
 		# give a test still 1 second, because system could be too busy
-		self.assertTrue(time.time() >= stime + 1 and time.time() <= stime + 2)
-		self.assertLogged(
-			'sleep 30 -- timed out after 1 seconds',
-			'sleep 30 -- timed out after 2 seconds'
-		)
+		self.assertTrue(time.time() >= stime + timeout and time.time() <= stime + timeout + 1)
+		self.assertLogged('sleep 30 -- timed out after')
 		self.assertLogged('sleep 30 -- killed with SIGTERM')
 
 	def testExecuteTimeoutWithNastyChildren(self):

From 542419acab7066768bd455abfef120569c7d94f1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 29 Sep 2016 20:51:20 +0200
Subject: [PATCH 198/623] filtertestcase: use shorter sleep (almost just for
 the context switch here)

---
 fail2ban/tests/filtertestcase.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 15495655..8e62531a 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -225,7 +225,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line
 		# Opened earlier, therefore must close it
 		fin.close()
 	# to give other threads possibly some time to crunch
-	time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
+	time.sleep(Utils.DEFAULT_SHORT_INTERVAL)
 	return fout
 
 
@@ -909,7 +909,7 @@ def get_monitor_failures_testcase(Filter_):
 
 			if interim_kill:
 				_killfile(None, self.name)
-				time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)				  # let them know
+				time.sleep(Utils.DEFAULT_SHORT_INTERVAL)				  # let them know
 
 			# now create a new one to override old one
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new',

From 62b8664175e6503979011abab98fe1ca61bf9cac Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 29 Sep 2016 21:02:37 +0200
Subject: [PATCH 199/623] speedup server start/stop (waiting for communicate,
 etc); extend server socket with timeouts, extend ping with timeout parameter;

---
 fail2ban/client/csocket.py               |  8 ++-
 fail2ban/client/fail2banclient.py        | 55 ++++++++++++------
 fail2ban/client/fail2banserver.py        |  8 ++-
 fail2ban/tests/fail2banclienttestcase.py | 74 +++++++++++++++++-------
 4 files changed, 105 insertions(+), 40 deletions(-)

diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py
index 45d58cc6..6b478460 100644
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@@ -32,10 +32,13 @@ import sys
 
 class CSocket:
 	
-	def __init__(self, sock="/var/run/fail2ban/fail2ban.sock"):
+	def __init__(self, sock="/var/run/fail2ban/fail2ban.sock", timeout=-1):
 		# Create an INET, STREAMing socket
 		#self.csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 		self.__csock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+		self.__deftout = self.__csock.gettimeout()
+		if timeout != -1:
+			self.settimeout(timeout)
 		#self.csock.connect(("localhost", 2222))
 		self.__csock.connect(sock)
 
@@ -50,6 +53,9 @@ class CSocket:
 		self.__csock.send(obj + CSPROTO.END)
 		return self.receive(self.__csock)
 
+	def settimeout(self, timeout):
+		self.__csock.settimeout(timeout if timeout != -1 else self.__deftout)
+
 	def close(self, sendEnd=True):
 		if not self.__csock:
 			return
diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 9bc1525e..331c5f34 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -36,6 +36,8 @@ from .beautifier import Beautifier
 from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \
 	logSys, exit, output
 
+from ..server.utils import Utils
+
 PROMPT = "fail2ban> "
 
 
@@ -69,8 +71,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		logSys.warning("Caught signal %d. Exiting" % signum)
 		exit(-1)
 
-	def __ping(self):
-		return self.__processCmd([["ping"]], False)
+	def __ping(self, timeout=0.1):
+		return self.__processCmd([["ping"] + ([timeout] if timeout != -1 else [])],
+			False, timeout=timeout)
 
 	@property
 	def beautifier(self):
@@ -79,7 +82,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		self._beautifier = Beautifier()
 		return self._beautifier
 
-	def __processCmd(self, cmd, showRet=True):
+	def __processCmd(self, cmd, showRet=True, timeout=-1):
 		client = None
 		try:
 			beautifier = self.beautifier
@@ -88,7 +91,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				beautifier.setInputCmd(c)
 				try:
 					if not client:
-						client = CSocket(self._conf["socket"])
+						client = CSocket(self._conf["socket"], timeout=timeout)
+					elif timeout != -1:
+						client.settimeout(timeout)
+					if self._conf["verbose"] > 2:
+						logSys.log(5, "CMD: %r", c)
 					ret = client.send(c)
 					if ret[0] == 0:
 						logSys.log(5, "OK : %r", ret[1])
@@ -101,10 +108,10 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						streamRet = False
 				except socket.error as e:
 					if showRet or self._conf["verbose"] > 1:
-						if showRet or c != ["ping"]:
-							self.__logSocketError()
+						if showRet or c[0] != "ping":
+							self.__logSocketError(e, c[0] == "ping")
 						else:
-							logSys.log(5, " -- ping failed -- %r", e)
+							logSys.log(5, " -- %s failed -- %r", c, e)
 					return False
 				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
@@ -125,14 +132,18 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				sys.stdout.flush()
 		return streamRet
 
-	def __logSocketError(self):
+	def __logSocketError(self, prevError="", errorOnly=False):
 		try:
 			if os.access(self._conf["socket"], os.F_OK): # pragma: no cover
 				# This doesn't check if path is a socket,
 				#  but socket.error should be raised
 				if os.access(self._conf["socket"], os.W_OK):
 					# Permissions look good, but socket.error was raised
-					logSys.error("Unable to contact server. Is it running?")
+					if errorOnly:
+						logSys.error(prevError)
+					else:
+						logSys.error("%sUnable to contact server. Is it running?", 
+							("[%s] " % prevError) if prevError else '')
 				else:
 					logSys.error("Permission denied to socket: %s,"
 								 " (you must be root)", self._conf["socket"])
@@ -188,7 +199,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# Start the server or just initialize started one:
 		try:
 			if background:
-				# Start server daemon as fork of client process:
+				# Start server daemon as fork of client process (or new process):
 				Fail2banServer.startServerAsync(self._conf)
 				# Send config stream to server:
 				if not self.__processStartStreamAfterWait(stream, False):
@@ -233,6 +244,11 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			logSys.log(5, '  client phase %s', phase)
 		if not stream:
 			return False
+		# wait a litle bit for phase "start-ready" before enter active waiting:
+		if phase is not None:
+			Utils.wait_for(lambda: phase.get('start-ready', None) is not None, 0.5, 0.001)
+			phase['configure'] = (True if stream else False)
+			logSys.log(5, '  client phase %s', phase)
 		# configure server with config stream:
 		ret = self.__processStartStreamAfterWait(stream, False)
 		if phase is not None:
@@ -293,7 +309,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 						return False
 					# stop options - jail name or --all
 					break
-			if self.__ping():
+			if self.__ping(timeout=-1):
 				if len(cmd) == 1:
 					jail = '--all'
 					ret, stream = self.readConfig()
@@ -311,6 +327,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				logSys.error("Could not find server")
 				return False
 
+		elif len(cmd) > 1 and cmd[0] == "ping":
+			return self.__processCmd([cmd], timeout=float(cmd[1]))
+
 		else:
 			return self.__processCmd([cmd])
 
@@ -342,21 +361,23 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# Wait for the server to start (the server has 30 seconds to answer ping)
 		starttime = time.time()
 		logSys.log(5, "__waitOnServer: %r", (alive, maxtime))
-		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping()
+		sltime = 0.0125 / 2
+		test = lambda: os.path.exists(self._conf["socket"]) and self.__ping(timeout=sltime)
 		with VisualWait(self._conf["verbose"]) as vis:
-			sltime = 0.0125 / 2
 			while self._alive:
 				runf = test()
 				if runf == alive:
 					return True
-				now = time.time()
+				waittime = time.time() - starttime
+				logSys.log(5, "  wait-time: %s", waittime)
 				# Wonderful visual :)
-				if now > starttime + 1:
+				if waittime > 1:
 					vis.heartbeat()
 				# f end time reached:
-				if now - starttime >= maxtime:
+				if waittime >= maxtime:
 					raise ServerExecutionException("Failed to start server")
-				sltime = min(sltime * 2, 0.5)
+				# first 200ms faster:
+				sltime = min(sltime * 2, 0.5 if waittime > 0.2 else 0.1)
 				time.sleep(sltime)
 		return False
 
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 85259e79..faf1732a 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -178,13 +178,17 @@ class Fail2banServer(Fail2banCmdLine):
 				logSys.debug('Configure via async client thread')
 				cli.configureServer(async=True, phase=phase)
 				# wait, do not continue if configuration is not 100% valid:
-				Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"])
+				Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001)
+				logSys.log(5, '  server phase %s', phase)
 				if not phase.get('start', False):
 					raise ServerExecutionException('Async configuration of server failed')
 
 			# Start server, daemonize it, etc.
 			pid = os.getpid()
 			server = Fail2banServer.startServerDirect(self._conf, background)
+			if not async:
+				phase['start-ready'] = True
+				logSys.log(5, '  server phase %s', phase)
 			# If forked - just exit other processes
 			if pid != os.getpid(): # pragma: no cover
 				os._exit(0)
@@ -193,7 +197,7 @@ class Fail2banServer(Fail2banCmdLine):
 
 			# wait for client answer "done":
 			if not async and cli:
-				Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"])
+				Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"], 0.001)
 				if not phase.get('done', False):
 					if server: # pragma: no cover
 						server.quit()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 8159a9f6..82164a80 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -184,6 +184,19 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 		"--timeout", str(fail2bancmdline.MAX_WAITTIME),
 	)
 
+def _get_pid_from_file(pidfile):
+	f = pid = None
+	try:
+		f = open(pidfile)
+		pid = f.read()
+		pid = re.match(r'\S+', pid).group()
+		return int(pid)
+	except Exception as e: # pragma: no cover
+		logSys.debug(e)
+	finally:
+		if f is not None:
+			f.close()
+	return pid
 
 def _kill_srv(pidfile):
 	logSys.debug("cleanup: %r", (pidfile, isdir(pidfile)))
@@ -193,23 +206,22 @@ def _kill_srv(pidfile):
 		if not isfile(pidfile): # pragma: no cover
 			pidfile = pjoin(piddir, "fail2ban.pid")
 
+	# output log in heavydebug (to see possible start errors):
+	if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
+		logfile = pjoin(piddir, "f2b.log")
+		if isfile(logfile):
+			_out_file(logfile)
+		else:
+			logSys.log(5, 'no logfile %r', logfile)
+
 	if not isfile(pidfile):
 		logSys.debug("cleanup: no pidfile for %r", piddir)
 		return True
 
-	f = pid = None
-	try:
-		logSys.debug("cleanup pidfile: %r", pidfile)
-		f = open(pidfile)
-		pid = f.read()
-		pid = re.match(r'\S+', pid).group()
-		pid = int(pid)
-	except Exception as e: # pragma: no cover
-		logSys.debug(e)
+	logSys.debug("cleanup pidfile: %r", pidfile)
+	pid = _get_pid_from_file(pidfile)
+	if pid is None: # pragma: no cover
 		return False
-	finally:
-		if f is not None:
-			f.close()
 
 	try:
 		logSys.debug("cleanup pid: %r", pid)
@@ -443,14 +455,18 @@ class Fail2banClientTest(Fail2banClientServerBase):
 	def testClientStartBackgroundCall(self, tmp):
 		global INTERACT
 		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
-		# start (in new process, using the same python version):
-		cmd = (sys.executable, pjoin(BIN, CLIENT))
-		logSys.debug('Start %s ...', cmd)
-		cmd = cmd + startparams + ("--async", "start",)
-		ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
-		self.assertTrue(len(ret) and ret[0])
-		# wait for server (socket and ready):
-		self._wait_for_srv(tmp, True, startparams=cmd)
+		# if fast, start server process from client started direct here:
+		if unittest.F2B.fast: # pragma: no cover
+			self.execSuccess(startparams + ("start",))
+		else:
+			# start (in new process, using the same python version):
+			cmd = (sys.executable, pjoin(BIN, CLIENT))
+			logSys.debug('Start %s ...', cmd)
+			cmd = cmd + startparams + ("--async", "start",)
+			ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True)
+			self.assertTrue(len(ret) and ret[0])
+			# wait for server (socket and ready):
+			self._wait_for_srv(tmp, True, startparams=cmd)
 		self.assertLogged("Server ready")
 		self.pruneLog()
 		try:
@@ -459,6 +475,24 @@ class Fail2banClientTest(Fail2banClientServerBase):
 			self.assertLogged("TEST-ECHO")
 			self.assertLogged("Exit with code 0")
 			self.pruneLog()
+			# test ping timeout:
+			self.execSuccess(startparams, "ping", "0.1")
+			self.assertLogged("Server replied: pong")
+			self.pruneLog()
+			# python 3 seems to bypass such short timeouts also, 
+			# so suspend/resume server process and test between it...
+			pid = _get_pid_from_file(pjoin(tmp, "f2b.pid"))
+			try:
+				# suspend:
+				os.kill(pid, signal.SIGSTOP); # or SIGTSTP?
+				time.sleep(Utils.DEFAULT_SHORT_INTERVAL)
+				# test ping with short timeout:
+				self.execFailed(startparams, "ping", "1e-10")
+			finally:
+				# resume:
+				os.kill(pid, signal.SIGCONT)
+			self.assertLogged("timed out")
+			self.pruneLog()
 			# interactive client chat with started server:
 			INTERACT += [
 				"echo INTERACT-ECHO",

From 2cfaf845ca9cb275cc8c6e8ca6a6c1c8b48a900d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 29 Sep 2016 21:23:37 +0200
Subject: [PATCH 200/623] standardize and normalize logging and verbosity
 formats, logging level etc between command lines (server, client,
 test-cases); test cases could pass (so increase) verbosity to the client (and
 furthermore client to the server also), usable for debug purposes resp.
 simplifying read of the log-file; custom and precise numeric log-levels can
 be given in test cases now;

---
 fail2ban/client/beautifier.py            | 21 +++---------
 fail2ban/client/fail2bancmdline.py       | 12 ++++---
 fail2ban/client/fail2banserver.py        |  2 ++
 fail2ban/helpers.py                      | 14 +++++++-
 fail2ban/server/server.py                | 19 +++++++----
 fail2ban/tests/fail2banclienttestcase.py | 22 +++++++-----
 fail2ban/tests/utils.py                  | 43 +++++++++++-------------
 7 files changed, 74 insertions(+), 59 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 2d2c0caf..df44afbb 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -22,7 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko"
 __license__ = "GPL"
 
 from ..exceptions import UnknownJailException, DuplicateJailException
-from ..helpers import getLogger
+from ..helpers import getLogger, logging
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -97,16 +97,7 @@ class Beautifier:
 				msg += "`- " + response
 			elif inC[1:2] == ['loglevel']:
 				msg = "Current logging level is "
-				if response == 1:
-					msg += "ERROR"
-				elif response == 2:
-					msg += "WARN"
-				elif response == 3:
-					msg += "INFO"
-				elif response == 4:
-					msg += "DEBUG"
-				else:
-					msg += repr(response)
+				msg += repr(logging.getLevelName(response) if isinstance(response, int) else response)
 			elif inC[1] == "dbfile":
 				if response is None:
 					msg = "Database currently disabled"
@@ -187,14 +178,12 @@ class Beautifier:
 					msg += ", ".join(response)
 		except Exception:
 			logSys.warning("Beautifier error. Please report the error")
-			logSys.error("Beautify " + repr(response) + " with "
-				+ repr(self.__inputCmd) + " failed")
-			msg += repr(response)
+			logSys.error("Beautify %r with %r failed", response, self.__inputCmd)
+			msg = repr(msg) + repr(response)
 		return msg
 
 	def beautifyError(self, response):
-		logSys.debug("Beautify (error) " + repr(response) + " with "
-					 + repr(self.__inputCmd))
+		logSys.debug("Beautify (error) %r with %r", response, self.__inputCmd)
 		msg = response
 		if isinstance(response, UnknownJailException):
 			msg = "Sorry but the jail '" + response.args[0] + "' does not exist"
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index e1bc5f1e..74236ab1 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -27,7 +27,7 @@ import sys
 
 from ..version import version
 from ..protocol import printFormatted
-from ..helpers import getLogger
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
 
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
@@ -200,8 +200,10 @@ class Fail2banCmdLine():
 					logSys.setLevel(logging.HEAVYDEBUG)
 				# Add the default logging handler to dump to stderr
 				logout = logging.StreamHandler(sys.stderr)
-				# set a format which is simpler for console use
-				formatter = logging.Formatter('%(levelname)-6s %(message)s')
+
+				# Custom log format for the verbose run (-1, because default verbosity here is 1):
+				fmt = getVerbosityFormat(verbose-1)
+				formatter = logging.Formatter(fmt)
 				# tell the handler to use this format
 				logout.setFormatter(formatter)
 				logSys.addHandler(logout)
@@ -218,8 +220,10 @@ class Fail2banCmdLine():
 
 			logSys.info("Using socket file %s", self._conf["socket"])
 
+			# Check log-level before start (or transmit to server), to prevent error in background:
+			llev = str2LogLevel(self._conf["loglevel"])
 			logSys.info("Using pid file %s, [%s] logging to %s",
-				self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"])
+				self._conf["pidfile"], logging.getLevelName(llev), self._conf["logtarget"])
 
 			if self._conf.get("dump", False):
 				ret, stream = self.readConfig()
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index faf1732a..aba8f8ca 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -94,6 +94,8 @@ class Fail2banServer(Fail2banCmdLine):
 			# Force the execution if needed.
 			if conf["force"]:
 				args.append("-x")
+			if conf["verbose"] > 1:
+				args.append("-" + "v"*(conf["verbose"]-1))
 			# Logging parameters:
 			for o in ('loglevel', 'logtarget', 'syslogsocket'):
 				args.append("--"+o)
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index bbed61cf..4f4426b0 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -134,11 +134,23 @@ def str2LogLevel(value):
 		if isinstance(value, int) or value.isdigit():
 			ll = int(value)
 		else:
-			ll = getattr(logging, value)
+			ll = getattr(logging, value.upper())
 	except AttributeError:
 		raise ValueError("Invalid log level %r" % value)
 	return ll
 
+def getVerbosityFormat(verbosity, fmt=' %(message)s'):
+	"""Custom log format for the verbose runs
+	"""
+	if verbosity > 1: # pragma: no cover
+		if verbosity > 3:
+			fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
+		if verbosity > 2:
+			fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
+		else:
+			fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
+	return fmt
+
 
 def excepthook(exctype, value, traceback):
 	"""Except hook used to log unhandled exceptions to Fail2Ban log
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 2ae96e15..501083ed 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -38,7 +38,7 @@ from .filter import FileFilter, JournalFilter
 from .transmitter import Transmitter
 from .asyncserver import AsyncServer, AsyncServerException
 from .. import version
-from ..helpers import getLogger, str2LogLevel, excepthook
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, excepthook
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -72,6 +72,7 @@ class Server:
 		self.__asyncServer = None
 		self.__logLevel = None
 		self.__logTarget = None
+		self.__verbose = None
 		self.__syslogSocket = None
 		self.__autoSyslogSocketPaths = {
 			'Darwin':  '/var/run/syslog',
@@ -111,6 +112,7 @@ class Server:
 			# We are daemon.
 		
 		# Set all logging parameters (or use default if not specified):
+		self.__verbose = conf.get("verbose", None)
 		self.setSyslogSocket(conf.get("syslogsocket", 
 			self.__syslogSocket if self.__syslogSocket is not None else DEF_SYSLOGSOCKET))
 		self.setLogLevel(conf.get("loglevel", 
@@ -542,10 +544,10 @@ class Server:
 				self.__logTarget = target
 				return True
 			# set a format which is simpler for console use
-			formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s")
+			fmt = "%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
 			if target == "SYSLOG":
 				# Syslog daemons already add date to the message.
-				formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)s %(message)s")
+				fmt = "%(name)s[%(process)d]: %(levelname)s %(message)s"
 				facility = logging.handlers.SysLogHandler.LOG_DAEMON
 				if self.__syslogSocket == "auto":
 					import platform
@@ -572,8 +574,8 @@ class Server:
 					open(target, "a").close()
 					hdlr = logging.handlers.RotatingFileHandler(target)
 				except IOError:
-					logSys.error("Unable to log to " + target)
-					logSys.info("Logging to previous target " + self.__logTarget)
+					logSys.error("Unable to log to %r", target)
+					logSys.info("Logging to previous target %r", self.__logTarget)
 					return False
 			# Removes previous handlers -- in reverse order since removeHandler
 			# alter the list in-place and that can confuses the iterable
@@ -592,8 +594,13 @@ class Server:
 					if (2, 6, 3) <= sys.version_info < (3,) or \
 							(3, 2) <= sys.version_info:
 						raise
+			# detailed format by deep log levels (as DEBUG=10):
+			if logger.getEffectiveLevel() <= logging.DEBUG: # pragma: no cover
+				if self.__verbose is None:
+					self.__verbose = logging.DEBUG - logger.getEffectiveLevel() + 1
+				fmt = getVerbosityFormat(self.__verbose-1)
 			# tell the handler to use this format
-			hdlr.setFormatter(formatter)
+			hdlr.setFormatter(logging.Formatter(fmt))
 			logger.addHandler(hdlr)
 			# Does not display this message at startup.
 			if self.__logTarget is not None:
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 82164a80..d8fc156b 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -174,13 +174,19 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 			"[DEFAULT]", "",
 			"",
 		)
-		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(pjoin(cfg, "fail2ban.conf"))
 			_out_file(pjoin(cfg, "jail.conf"))
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
+	vvv, llev = (), "INFO"
+	if unittest.F2B.log_level < logging.INFO: # pragma: no cover
+		llev = str(unittest.F2B.log_level)
+		if unittest.F2B.verbosity > 1:
+			vvv = ("-" + "v"*unittest.F2B.verbosity,)
+	llev = vvv + ("--loglevel", llev)
 	return (
 		"-c", cfg, "-s", pjoin(tmp, "f2b.sock"), "-p", pjoin(tmp, "f2b.pid"),
-		"-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto",
+		"--logtarget", logtarget,) + llev + ("--syslogsocket", "auto",
 		"--timeout", str(fail2bancmdline.MAX_WAITTIME),
 	)
 
@@ -726,7 +732,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
 				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
 			)
-			if DefLogSys.level <= logging.DEBUG:  # if DEBUG
+			if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
 				_out_file(fn)
 
 		def _write_jail_cfg(enabled=(1, 2), actions=()):
@@ -754,7 +760,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
-			if DefLogSys.level <= logging.DEBUG:  # if DEBUG
+			if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
 				_out_file(pjoin(cfg, "jail.conf"))
 
 		# create default test actions:
@@ -768,7 +774,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		
 		# reload and wait for ban:
 		self.pruneLog("[test-phase 1a]")
-		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertLogged(
@@ -786,7 +792,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 1b]")
 		_write_jail_cfg(actions=[1,2])
 		_write_file(test1log, "w+")
-		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
@@ -848,7 +854,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 +
 		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.8: test 2",) * 3
 		))
-		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test2log)
 		# test all will be found in jail1 and one in jail2:
 		self.assertLogged(
@@ -947,7 +953,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			(str(int(MyTime.time())) + "   error 403 from 192.0.2.5: test 5",) * 3 +
 			(str(int(MyTime.time())) + " failure 401 from 192.0.2.6: test 5",) * 3
 		))
-		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test1log)
 		self.assertLogged(
 			"6 ticket(s) in 'test-jail1",
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 3c044951..77210088 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -37,7 +37,7 @@ import unittest
 from cStringIO import StringIO
 from functools import wraps
 
-from ..helpers import getLogger
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
 from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
@@ -82,9 +82,8 @@ def getOptParser(doc=""):
 				version="%prog " + version)
 
 	p.add_options([
-		Option('-l', "--log-level", type="choice",
+		Option('-l', "--log-level",
 			   dest="log_level",
-			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
 		Option('-v', "--verbosity", action="store",
@@ -122,22 +121,11 @@ def initProcess(opts):
 	global logSys
 	logSys = getLogger("fail2ban")
 
-	# Numerical level of verbosity corresponding to a log "level"
-	verbosity = opts.verbosity
-	if verbosity is None:
-		verbosity = {'heavydebug': 4,
-					 'debug': 3,
-					 'info': 2,
-					 'notice': 2,
-					 'warning': 1,
-					 'error': 1,
-					 'critical': 0,
-					 None: 1}[opts.log_level]
-		opts.verbosity = verbosity
-
+	llev = None
 	if opts.log_level is not None: # pragma: no cover
 		# so we had explicit settings
-		logSys.setLevel(getattr(logging, opts.log_level.upper()))
+		llev = str2LogLevel(opts.log_level)
+		logSys.setLevel(llev)
 	else: # pragma: no cover
 		# suppress the logging but it would leave unittests' progress dots
 		# ticking, unless like with '-l critical' which would be silent
@@ -145,6 +133,19 @@ def initProcess(opts):
 		logSys.setLevel(logging.CRITICAL)
 	opts.log_level = logSys.level
 
+	# Numerical level of verbosity corresponding to a given log "level"
+	verbosity = opts.verbosity
+	if verbosity is None:
+		verbosity = (
+			1 if llev is None else \
+			4 if llev <= logging.HEAVYDEBUG else \
+			3 if llev <= logging.DEBUG else \
+			2 if llev <= min(logging.INFO, logging.NOTICE) else \
+			1 if llev <= min(logging.WARNING, logging.ERROR) else \
+			0 # if llev <= logging.CRITICAL
+		)
+		opts.verbosity = verbosity
+
 	# Add the default logging handler
 	stdout = logging.StreamHandler(sys.stdout)
 
@@ -157,13 +158,7 @@ def initProcess(opts):
 		Formatter = logging.Formatter
 
 	# Custom log format for the verbose tests runs
-	if verbosity > 1: # pragma: no cover
-		if verbosity > 3:
-			fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt
-		if verbosity > 2:
-			fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
-		else:
-			fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
+	fmt = getVerbosityFormat(verbosity, fmt)
 
 	#
 	stdout.setFormatter(Formatter(fmt))

From 77ec9df67894d38bc11a705d6958c060abc4d135 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 10:01:21 +0200
Subject: [PATCH 201/623] standardize and normalize verbosity parameters for
 fail2ban-regex / fail2ban-testcases (-v ... -vvvv, or --verbosity=0..4)

---
 fail2ban/client/fail2banregex.py | 25 ++++++++++++-------------
 fail2ban/tests/utils.py          |  6 ++++--
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index f99f9f01..c05006f7 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -51,7 +51,7 @@ from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import RegexException
 
-from ..helpers import FormatterWithTraceBack, getLogger, PREFER_ENC
+from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, PREFER_ENC
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
@@ -134,13 +134,16 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 		Option("-m", "--journalmatch",
 			   help="journalctl style matches overriding filter file. "
 			   "\"systemd-journal\" only"),
-		Option('-l', "--log-level", type="choice",
+		Option('-l', "--log-level",
 			   dest="log_level",
-			   choices=('heavydebug', 'debug', 'info', 'notice', 'warning', 'error', 'critical'),
 			   default=None,
 			   help="Log level for the Fail2Ban logger to use"),
-		Option("-v", "--verbose", action='store_true',
-			   help="Be verbose in output"),
+		Option('-v', '--verbose', action="count", dest="verbose",
+			   default=None,
+			   help="Increase verbosity"),
+		Option("--verbosity", action="store", dest="verbose", type=int,
+			   default=None,
+			   help="Set numerical level of verbosity (0..4)"),
 		Option("-D", "--debuggex", action='store_true',
 			   help="Produce debuggex.com urls for debugging there"),
 		Option("--print-no-missed", action='store_true',
@@ -579,17 +582,17 @@ def exec_command_line():
 	# TODO: taken from -testcases -- move common functionality somewhere
 	if opts.log_level is not None:
 		# so we had explicit settings
-		logSys.setLevel(getattr(logging, opts.log_level.upper()))
+		logSys.setLevel(str2LogLevel(opts.log_level))
 	else:
 		# suppress the logging but it would leave unittests' progress dots
 		# ticking, unless like with '-l critical' which would be silent
 		# unless error occurs
-		logSys.setLevel(getattr(logging, 'CRITICAL'))
+		logSys.setLevel(logging.CRITICAL)
 
 	# Add the default logging handler
 	stdout = logging.StreamHandler(sys.stdout)
 
-	fmt = 'D: %(message)s'
+	fmt = '%(levelname)-1.1s: %(message)s' if opts.verbose <= 1 else '%(message)s'
 
 	if opts.log_traceback:
 		Formatter = FormatterWithTraceBack
@@ -598,11 +601,7 @@ def exec_command_line():
 		Formatter = logging.Formatter
 
 	# Custom log format for the verbose tests runs
-	if opts.verbose:
-		stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt))
-	else:
-		# just prefix with the space
-		stdout.setFormatter(Formatter(fmt))
+	stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt)))
 	logSys.addHandler(stdout)
 
 	fail2banRegex = Fail2banRegex(opts)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 77210088..02f417c8 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -86,8 +86,10 @@ def getOptParser(doc=""):
 			   dest="log_level",
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
-		Option('-v', "--verbosity", action="store",
-			   dest="verbosity", type=int,
+		Option('-v', action="count", dest="verbosity",
+			   default=None,
+			   help="Increase verbosity"),
+		Option("--verbosity", action="store", dest="verbosity", type=int,
 			   default=None,
 			   help="Set numerical level of verbosity (0..4)"),
 		Option("--log-direct", action="store_false",

From 057f2f3c56c7d2c8396a05bc834ab95abcba0d04 Mon Sep 17 00:00:00 2001
From: Niklas Fiekas <niklas.fiekas@backscattering.de>
Date: Fri, 30 Sep 2016 11:08:07 +0200
Subject: [PATCH 202/623] make the ipv6 host regex greedy

Previously the regex was lazily matching ``2606:2800:220:1:248:1893:25c8:1946``
as ``2606:2800:220:1:248:1893:25c8:1``.
---
 fail2ban/server/failregex.py              | 2 +-
 fail2ban/tests/files/logs/nginx-limit-req | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 7c512609..56dc7b14 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -77,7 +77,7 @@ class Regex:
 		regex = regex.replace("<F-IP4/>", r); # closed
 		r_host.append(r)
 		# separated ipv6:
-		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}?|(?<=:):))"""
+		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))"""
 		regex = regex.replace("<IP6>", r); # self closed
 		regex = regex.replace("<F-IP6/>", r); # closed
 		r_host.append(r"""\[?%s\]?""" % (r,)); # enclose ipv6 in optional [] in host-regex
diff --git a/fail2ban/tests/files/logs/nginx-limit-req b/fail2ban/tests/files/logs/nginx-limit-req
index 68f1b239..9a77b45f 100644
--- a/fail2ban/tests/files/logs/nginx-limit-req
+++ b/fail2ban/tests/files/logs/nginx-limit-req
@@ -4,3 +4,9 @@
 
 # failJSON: { "time": "2015-10-29T19:24:05", "match": true , "host": "192.0.2.0" }
 2015/10/29 19:24:05 [error] 12684#12684: *22174 limiting requests, excess: 1.495 by zone "one", client: 192.0.2.0, server: example.com, request: "GET /index.php HTTP/1.1", host: "example.com", referrer: "https://example.com"
+
+# failJSON: { "time": "2016-09-30T08:36:06", "match": true, "host": "13.123.1.123" }
+2016/09/30 08:36:06 [error] 22923#0: *4758725916 limiting requests, excess: 15.243 by zone "one", client: 13.123.1.123, server: example.com, request: "GET / HTTP/1.1", host: "example.com"
+
+# failJSON: { "time": "2016-09-30T08:36:06", "match": true, "host": "2606:2800:220:1:248:1893:25c8:1946" }
+2016/09/30 08:36:06 [error] 22923#0: *4758725916 limiting requests, excess: 15.243 by zone "one", client: 2606:2800:220:1:248:1893:25c8:1946, server: example.com, request: "GET / HTTP/1.1", host: "example.com"

From 9a7c753372c591ea329d30d7fc28db014b73b38e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 11:26:49 +0200
Subject: [PATCH 203/623] fixes method-related filter for tests of suite loaded
 with loadTestsFromName (they may be a suite self)

---
 fail2ban/tests/utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 02f417c8..ea051872 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -330,6 +330,11 @@ def gatherTests(regexps=None, opts=None):
 			def addTest(self, suite):
 				matched = []
 				for test in suite:
+					# test of suite loaded with loadTestsFromName may be a suite self:
+					if isinstance(test, unittest.TestSuite): # pragma: no cover
+						self.addTest(test)
+						continue
+					# filter by regexp:
 					s = str(test)
 					for r in self._regexps:
 						m = r.search(s)

From 8b0f6c54134f86f3fcc096feeea89492693398dd Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 11:28:21 +0200
Subject: [PATCH 204/623] badips test cases check availability of badips
 service (and skip this tests if it not available)

---
 config/action.d/badips.py              | 12 ++++++++-
 fail2ban/server/actions.py             | 36 +++++++++++++++-----------
 fail2ban/tests/action_d/test_badips.py | 14 +++++++++-
 3 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/config/action.d/badips.py b/config/action.d/badips.py
index 4bc879a1..473fbf33 100644
--- a/config/action.d/badips.py
+++ b/config/action.d/badips.py
@@ -34,7 +34,7 @@ else:
 from fail2ban.server.actions import ActionBase
 
 
-class BadIPsAction(ActionBase):
+class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 	"""Fail2Ban action which reports bans to badips.com, and also
 	blacklist bad IPs listed on badips.com by using another action's
 	ban method.
@@ -105,6 +105,16 @@ class BadIPsAction(ActionBase):
 		# Used later for threading.Timer for updating badips
 		self._timer = None
 
+	@staticmethod
+	def isAvailable(timeout=1):
+		try:
+			response = urlopen(Request("/".join([BadIPsAction._badips]),
+					headers={'User-Agent': "Fail2Ban"}), timeout=timeout)
+			return True, ''
+		except Exception as e: # pragma: no cover
+			return False, e
+
+
 	def getCategories(self, incParents=False):
 		"""Get badips.com categories.
 
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index eed481d7..782cd3de 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -85,6 +85,26 @@ class Actions(JailThread, Mapping):
 		## The ban manager.
 		self.__banManager = BanManager()
 
+	@staticmethod
+	def _load_python_module(pythonModule):
+		pythonModuleName = os.path.splitext(
+			os.path.basename(pythonModule))[0]
+		if sys.version_info >= (3, 3):
+			mod = importlib.machinery.SourceFileLoader(
+				pythonModuleName, pythonModule).load_module()
+		else:
+			mod = imp.load_source(
+				pythonModuleName, pythonModule)
+		if not hasattr(mod, "Action"):
+			raise RuntimeError(
+				"%s module does not have 'Action' class" % pythonModule)
+		elif not issubclass(mod.Action, ActionBase):
+			raise RuntimeError(
+				"%s module %s does not implement required methods" % (
+					pythonModule, mod.Action.__name__))
+		return mod
+
+
 	def add(self, name, pythonModule=None, initOpts=None, reload=False):
 		"""Adds a new action.
 
@@ -127,21 +147,7 @@ class Actions(JailThread, Mapping):
 		if pythonModule is None:
 			action = CommandAction(self._jail, name)
 		else:
-			pythonModuleName = os.path.splitext(
-				os.path.basename(pythonModule))[0]
-			if sys.version_info >= (3, 3):
-				customActionModule = importlib.machinery.SourceFileLoader(
-					pythonModuleName, pythonModule).load_module()
-			else:
-				customActionModule = imp.load_source(
-					pythonModuleName, pythonModule)
-			if not hasattr(customActionModule, "Action"):
-				raise RuntimeError(
-					"%s module does not have 'Action' class" % pythonModule)
-			elif not issubclass(customActionModule.Action, ActionBase):
-				raise RuntimeError(
-					"%s module %s does not implement required methods" % (
-						pythonModule, customActionModule.Action.__name__))
+			customActionModule = self._load_python_module(pythonModule)
 			action = customActionModule.Action(self._jail, name, **initOpts)
 		self._actions[name] = action
 
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 2b9090cc..64db44e7 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -24,9 +24,12 @@ import sys
 from ..dummyjail import DummyJail
 from ..utils import CONFIG_DIR
 
-if sys.version_info >= (2,7):
+if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 	class BadIPsActionTest(unittest.TestCase):
 
+		available = True, None
+		modAction = None
+		
 		def setUp(self):
 			"""Call before every test case."""
 			unittest.F2B.SkipIfNoNetwork()
@@ -36,6 +39,15 @@ if sys.version_info >= (2,7):
 			self.jail.actions.add("test")
 
 			pythonModule = os.path.join(CONFIG_DIR, "action.d", "badips.py")
+
+			# check availability (once if not alive, used shorter timeout as in test cases):
+			if BadIPsActionTest.available[0]:
+				if not BadIPsActionTest.modAction:
+					BadIPsActionTest.modAction = self.jail.actions._load_python_module(pythonModule).Action
+				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 10)
+			if not BadIPsActionTest.available[0]:
+				raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1])
+
 			self.jail.actions.add("badips", pythonModule, initOpts={
 				'category': "ssh",
 				'banaction': "test",

From 06674bb98981f89c77c435a25528ce90c02670a6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 12:26:31 +0200
Subject: [PATCH 205/623] use common regex for IP addresses (removed code
 duplication)

---
 fail2ban/server/failregex.py | 5 +++--
 fail2ban/server/ipdns.py     | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 56dc7b14..7c51ddb8 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -25,6 +25,7 @@ import re
 import sre_constants
 import sys
 
+from .ipdns import IPAddr
 
 ##
 # Regular expression class.
@@ -72,12 +73,12 @@ class Regex:
 	def _resolveHostTag(regex, useDns="yes"):
 		# separated ipv4:
 		r_host = []
-		r = r"""(?:::f{4,6}:)?(?P<ip4>(?:\d{1,3}\.){3}\d{1,3})"""
+		r = r"""(?:::f{4,6}:)?(?P<ip4>%s)""" % (IPAddr.IP_4_RE,)
 		regex = regex.replace("<IP4>", r); # self closed
 		regex = regex.replace("<F-IP4/>", r); # closed
 		r_host.append(r)
 		# separated ipv6:
-		r = r"""(?P<ip6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))"""
+		r = r"""(?P<ip6>%s)""" % (IPAddr.IP_6_RE,)
 		regex = regex.replace("<IP6>", r); # self closed
 		regex = regex.replace("<F-IP6/>", r); # closed
 		r_host.append(r"""\[?%s\]?""" % (r,)); # enclose ipv6 in optional [] in host-regex
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index cba08d0c..f8db6a04 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -124,8 +124,11 @@ class DNSUtils:
 class IPAddr(object):
 	"""Encapsulate functionality for IPv4 and IPv6 addresses
 	"""
+
+	IP_4_RE = r"""(?:\d{1,3}\.){3}\d{1,3}"""
+	IP_6_RE = r"""(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):)"""
 	IP_4_6_CRE = re.compile(
-	  r"""^(?:(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?)$""")
+	  r"""^(?:(?P<IPv4>%s)|\[?(?P<IPv6>%s)\]?)$""" % (IP_4_RE, IP_6_RE))
 	# An IPv4 compatible IPv6 to be reused (see below)
 	IP6_4COMPAT = None
 

From 9bf8985e2a317bfb2acad660cebe9808e108ee70 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 12:31:57 +0200
Subject: [PATCH 206/623] nginx-limit-req.conf: more precise failregex
 (word-boundary if `<HOST>` should be non-greedy for some reasons)

---
 config/filter.d/nginx-limit-req.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf
index 589d3d78..13cf75c2 100644
--- a/config/filter.d/nginx-limit-req.conf
+++ b/config/filter.d/nginx-limit-req.conf
@@ -39,7 +39,7 @@ ngx_limit_req_zones = [^"]+
 # failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
 
 # Shortly, much faster and stable version of regexp:
-failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>
+failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,
 
 ignoreregex = 
 

From 276759b6c27673be08f84d0997cb107a4f9a0e4c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 13:18:19 +0200
Subject: [PATCH 207/623] ExtendedCymruInfo code review and availability check
 in test cases; max sleep time check of too long sleep increased to 1 second

---
 fail2ban/server/banmanager.py        | 24 ++++++++++--------------
 fail2ban/tests/banmanagertestcase.py | 19 +++++++++++++------
 fail2ban/tests/utils.py              |  2 +-
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 9aa69f5a..10578945 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -143,10 +143,12 @@ class BanManager:
 			return_dict["country"].append("error")
 			return_dict["rir"].append("error")
 			return return_dict
-		self.__lock.acquire()
+		# get ips in lock:
+		with self.__lock:
+			banIPs = [banData.getIP() for banData in self.__banList.values()]
+		# get cymru info:
 		try:
-			for banData in self.__banList.values():
-				ip = banData.getIP()
+			for ip in banIPs:
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
 				question = ip.getPTR(
 					"origin.asn.cymru.com" if ip.isIPv4
@@ -170,14 +172,17 @@ class BanManager:
 				except dns.exception.DNSException as dnse:
 					logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question)
 					logSys.exception(dnse)
+					return_dict["error"] = dnse
+					break
 				except Exception as e:
 					logSys.error("Unhandled Exception querying Cymru for %s TXT" % question)
 					logSys.exception(e)
+					return_dict["error"] = e
+					break
 		except Exception as e:
 			logSys.error("Failure looking up extended Cymru info")
 			logSys.exception(e)
-		finally:
-			self.__lock.release()
+			return_dict["error"] = e
 		return return_dict
 
 	##
@@ -188,15 +193,12 @@ class BanManager:
 	# @return list of Banned ASNs
 
 	def geBanListExtendedASN(self, cymru_info):
-		self.__lock.acquire()
 		try:
 			return [asn for asn in cymru_info["asn"]]
 		except Exception as e:
 			logSys.error("Failed to lookup ASN")
 			logSys.exception(e)
 			return []
-		finally:
-			self.__lock.release()
 
 	##
 	# Returns list of Banned Countries from Cymru info
@@ -206,15 +208,12 @@ class BanManager:
 	# @return list of Banned Countries
 
 	def geBanListExtendedCountry(self, cymru_info):
-		self.__lock.acquire()
 		try:
 			return [country for country in cymru_info["country"]]
 		except Exception as e:
 			logSys.error("Failed to lookup Country")
 			logSys.exception(e)
 			return []
-		finally:
-			self.__lock.release()
 
 	##
 	# Returns list of Banned RIRs from Cymru info
@@ -224,15 +223,12 @@ class BanManager:
 	# @return list of Banned RIRs
 
 	def geBanListExtendedRIR(self, cymru_info):
-		self.__lock.acquire()
 		try:
 			return [rir for rir in cymru_info["rir"]]
 		except Exception as e:
 			logSys.error("Failed to lookup RIR")
 			logSys.exception(e)
 			return []
-		finally:
-			self.__lock.release()
 
 	##
 	# Create a ban ticket.
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index b627599d..d557f16e 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -147,8 +147,15 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		"""Call after every test case."""
 		pass
 
-	def testCymruInfo(self):
+	def _getBanListExtendedCymruInfo(self):
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		if cymru_info.get("error"): # pragma: no cover - availability
+			raise unittest.SkipTest('Skip test because service is not available: %s' % cymru_info["error"])
+		return cymru_info
+
+
+	def testCymruInfo(self):
+		cymru_info = self._getBanListExtendedCymruInfo()
 		self.assertDictEqual(cymru_info,
 						  {"asn": [self.__asn],
 						   "country": [self.__country],
@@ -156,17 +163,17 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 
 	def testCymruInfoASN(self):
 		self.assertEqual(
-			self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()),
+			self.__banManager.geBanListExtendedASN(self._getBanListExtendedCymruInfo()),
 			[self.__asn])
 
 	def testCymruInfoCountry(self):
 		self.assertEqual(
-			self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()),
+			self.__banManager.geBanListExtendedCountry(self._getBanListExtendedCymruInfo()),
 			[self.__country])
 
 	def testCymruInfoRIR(self):
 		self.assertEqual(
-			self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()),
+			self.__banManager.geBanListExtendedRIR(self._getBanListExtendedCymruInfo()),
 			[self.__rir])
 
 	def testCymruInfoNxdomain(self):
@@ -175,7 +182,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		# non-existing IP
 		ticket = BanTicket("0.0.0.0", 1167605999.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
-		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		cymru_info = self._getBanListExtendedCymruInfo()
 		self.assertDictEqual(cymru_info,
 						  {"asn": ["nxdomain"],
 						   "country": ["nxdomain"],
@@ -186,7 +193,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		# and new ones
 		ticket = BanTicket("10.0.0.0", 1167606000.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
-		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		cymru_info = self._getBanListExtendedCymruInfo()
 		self.assertDictEqual(dict((k, sorted(v)) for k, v in cymru_info.iteritems()),
 						  {"asn": sorted(["nxdomain", "4565",]),
 						   "country": sorted(["nxdomain", "unknown"]),
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index ea051872..0f5ea9bd 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -248,7 +248,7 @@ def initTests(opts):
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):
-			if (v > Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
+			if v > min(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
 				raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v)
 			_org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME))
 		time.sleep = _new_sleep

From c2d2e79b0d48bf66b04c3772c2419f30a4b1f9db Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 17:01:06 +0200
Subject: [PATCH 208/623] ExtendedCymruInfo: better availability check (code
 review and timeout's); max sleep time check of too long sleep increased to 1
 second (typo fix)

---
 fail2ban/server/banmanager.py        | 60 +++++++++++++++++-----------
 fail2ban/tests/banmanagertestcase.py | 10 ++++-
 fail2ban/tests/utils.py              |  2 +-
 3 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 10578945..054ae4bb 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -28,7 +28,7 @@ from threading import Lock
 
 from .ticket import BanTicket
 from .mytime import MyTime
-from ..helpers import getLogger
+from ..helpers import getLogger, logging
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -132,17 +132,23 @@ class BanManager:
 	#
 	# @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs
 
-	def getBanListExtendedCymruInfo(self):
+	def getBanListExtendedCymruInfo(self, timeout=10):
 		return_dict = {"asn": [], "country": [], "rir": []}
-		try:
-			import dns.exception
-			import dns.resolver
-		except ImportError:
-			logSys.error("dnspython package is required but could not be imported")
-			return_dict["asn"].append("error")
-			return_dict["country"].append("error")
-			return_dict["rir"].append("error")
-			return return_dict
+		if not hasattr(self, 'dnsResolver'):
+			try:
+				import dns.exception
+				import dns.resolver
+				resolver = dns.resolver.Resolver()
+				resolver.lifetime = timeout
+				resolver.timeout = timeout / 2
+				self.dnsResolver = resolver
+			except ImportError: # pragma: no cover
+				logSys.error("dnspython package is required but could not be imported")
+				return_dict["error"] = repr(e)
+				return_dict["asn"].append("error")
+				return_dict["country"].append("error")
+				return_dict["rir"].append("error")
+				return return_dict
 		# get ips in lock:
 		with self.__lock:
 			banIPs = [banData.getIP() for banData in self.__banList.values()]
@@ -155,7 +161,10 @@ class BanManager:
 					else "origin6.asn.cymru.com"
 				)
 				try:
-					answers = dns.resolver.query(question, "TXT")
+					resolver = self.dnsResolver
+					answers = resolver.query(question, "TXT")
+					if not answers:
+						raise ValueError("No data retrieved")
 					for rdata in answers:
 						asn, net, country, rir, changed =\
 							[answer.strip("'\" ") for answer in rdata.to_text().split("|")]
@@ -169,20 +178,23 @@ class BanManager:
 					return_dict["asn"].append("nxdomain")
 					return_dict["country"].append("nxdomain")
 					return_dict["rir"].append("nxdomain")
-				except dns.exception.DNSException as dnse:
-					logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question)
-					logSys.exception(dnse)
-					return_dict["error"] = dnse
+				except (dns.exception.DNSException, dns.resolver.NoNameservers, dns.exception.Timeout) as dnse: # pragma: no cover
+					logSys.error("DNSException %r querying Cymru for %s TXT", dnse, question)
+					if logSys.level <= logging.DEBUG:
+						logSys.exception(dnse)
+					return_dict["error"] = repr(dnse)
 					break
-				except Exception as e:
-					logSys.error("Unhandled Exception querying Cymru for %s TXT" % question)
-					logSys.exception(e)
-					return_dict["error"] = e
+				except Exception as e: # pragma: no cover
+					logSys.error("Unhandled Exception %r querying Cymru for %s TXT", e, question)
+					if logSys.level <= logging.DEBUG:
+						logSys.exception(e)
+					return_dict["error"] = repr(e)
 					break
-		except Exception as e:
-			logSys.error("Failure looking up extended Cymru info")
-			logSys.exception(e)
-			return_dict["error"] = e
+		except Exception as e: # pragma: no cover
+			logSys.error("Failure looking up extended Cymru info: %s", e)
+			if logSys.level <= logging.DEBUG:
+				logSys.exception(e)
+			return_dict["error"] = repr(e)
 		return return_dict
 
 	##
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index d557f16e..4d964425 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -147,9 +147,17 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		"""Call after every test case."""
 		pass
 
+	available = True, None
+
 	def _getBanListExtendedCymruInfo(self):
-		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		tc = StatusExtendedCymruInfo
+		if tc.available[0]:
+			cymru_info = self.__banManager.getBanListExtendedCymruInfo(
+				timeout=(2 if unittest.F2B.fast else 20))
+		else:
+			cymru_info = tc.available[1]
 		if cymru_info.get("error"): # pragma: no cover - availability
+			tc.available = False, cymru_info
 			raise unittest.SkipTest('Skip test because service is not available: %s' % cymru_info["error"])
 		return cymru_info
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 0f5ea9bd..f60dfd1f 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -248,7 +248,7 @@ def initTests(opts):
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):
-			if v > min(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
+			if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
 				raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v)
 			_org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME))
 		time.sleep = _new_sleep

From 973ac9a45c6f4838d5e6be5b2e58e67ce7bfdeb7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 12:18:55 +0200
Subject: [PATCH 209/623] amend to c2d2e79b0d48bf66b04c3772c2419f30a4b1f9db:
 missing error variable in import block; additionally fixes forgotten skip for
 cymru server case, if --no-network specified); Closes #1568

---
 fail2ban/server/banmanager.py    | 2 +-
 fail2ban/tests/servertestcase.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 054ae4bb..9865c2d3 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -142,7 +142,7 @@ class BanManager:
 				resolver.lifetime = timeout
 				resolver.timeout = timeout / 2
 				self.dnsResolver = resolver
-			except ImportError: # pragma: no cover
+			except ImportError as e: # pragma: no cover
 				logSys.error("dnspython package is required but could not be imported")
 				return_dict["error"] = repr(e)
 				return_dict["asn"].append("error")
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ebc39d9b..56c85e94 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -559,6 +559,7 @@ class Transmitter(TransmitterBase):
 		)
 
 	def testJailStatusCymru(self):
+		unittest.F2B.SkipIfNoNetwork()
 		try:
 			import dns.exception
 			import dns.resolver

From fa8184d4ccee4e3314a0010dcc371eaf2785633a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 15:01:33 +0200
Subject: [PATCH 210/623] fixes deprecated DNSUtils.IsValidIP in fakegooglebot
 ignore command + test covered now; Closes #1559

---
 .../ignorecommands/apache-fakegooglebot       | 27 +++++++++--------
 fail2ban/server/actions.py                    | 17 ++---------
 fail2ban/server/utils.py                      | 17 +++++++++++
 fail2ban/tests/filtertestcase.py              | 30 +++++++++++++++++++
 4 files changed, 65 insertions(+), 26 deletions(-)

diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot
index 68a2cb5b..3c443251 100755
--- a/config/filter.d/ignorecommands/apache-fakegooglebot
+++ b/config/filter.d/ignorecommands/apache-fakegooglebot
@@ -5,31 +5,34 @@
 # presence of host and cut commands
 #
 import sys
+from fail2ban.server.ipdns import DNSUtils, IPAddr
 
 def process_args(argv):
     if len(argv) != 2:
-       sys.stderr.write("Please provide a single IP as an argument. Got: %s\n"
+       raise ValueError("Please provide a single IP as an argument. Got: %s\n"
                         % (argv[1:]))
-       sys.exit(2)
-
     ip = argv[1]
 
-    from fail2ban.server.ipdns import DNSUtils
-    if not DNSUtils.isValidIP(ip):
-       sys.stderr.write("Argument must be a single valid IP. Got: %s\n"
+    if not IPAddr(ip).isValid:
+       raise ValueError("Argument must be a single valid IP. Got: %s\n"
                         % ip)
-       sys.exit(3)
     return ip
 
+google_ips = None
+
 def is_googlebot(ip):
     import re
-    from fail2ban.server.ipdns import DNSUtils
 
     host = DNSUtils.ipToName(ip)
     if not host or not re.match('.*\.google(bot)?\.com$', host):
-       sys.exit(1)
+       return False
     host_ips = DNSUtils.dnsToIp(host)
-    sys.exit(0 if ip in host_ips else 1)
+    return (ip in host_ips)
 
-if __name__ == '__main__':
-    is_googlebot(process_args(sys.argv))
+if __name__ == '__main__': # pragma: no cover
+    try:
+      ret = is_googlebot(process_args(sys.argv))
+    except ValueError as e:
+      sys.stderr.write(str(e))
+      sys.exit(2)
+    sys.exit(0 if ret else 1)
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 782cd3de..dcfe09f6 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -28,10 +28,6 @@ import logging
 import os
 import sys
 import time
-if sys.version_info >= (3, 3):
-	import importlib.machinery
-else:
-	import imp
 from collections import Mapping
 try:
 	from collections import OrderedDict
@@ -87,18 +83,11 @@ class Actions(JailThread, Mapping):
 
 	@staticmethod
 	def _load_python_module(pythonModule):
-		pythonModuleName = os.path.splitext(
-			os.path.basename(pythonModule))[0]
-		if sys.version_info >= (3, 3):
-			mod = importlib.machinery.SourceFileLoader(
-				pythonModuleName, pythonModule).load_module()
-		else:
-			mod = imp.load_source(
-				pythonModuleName, pythonModule)
-		if not hasattr(mod, "Action"):
+		mod = Utils.load_python_module(pythonModule)
+		if not hasattr(mod, "Action"): # pragma: no cover
 			raise RuntimeError(
 				"%s module does not have 'Action' class" % pythonModule)
-		elif not issubclass(mod.Action, ActionBase):
+		elif not issubclass(mod.Action, ActionBase): # pragma: no cover
 			raise RuntimeError(
 				"%s module %s does not implement required methods" % (
 					pythonModule, mod.Action.__name__))
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 7c13631e..6ed3d8fa 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -30,6 +30,11 @@ import sys
 import time
 from ..helpers import getLogger, uni_decode
 
+if sys.version_info >= (3, 3):
+	import importlib.machinery
+else:
+	import imp
+
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
@@ -290,3 +295,15 @@ class Utils():
 				return True
 			else:
 				return False
+
+	@staticmethod
+	def load_python_module(pythonModule):
+		pythonModuleName = os.path.splitext(
+			os.path.basename(pythonModule))[0]
+		if sys.version_info >= (3, 3):
+			mod = importlib.machinery.SourceFileLoader(
+				pythonModuleName, pythonModule).load_module()
+		else:
+			mod = imp.load_source(
+				pythonModuleName, pythonModule)
+		return mod
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 8e62531a..2b57ce47 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -48,6 +48,9 @@ from .dummyjail import DummyJail
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
+STOCK_CONF_DIR = "config"
+STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR, 'fail2ban.conf'))
+
 
 # yoh: per Steven Hiscocks's insight while troubleshooting
 # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836
@@ -409,6 +412,33 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11"))
 		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13"))
 
+	def testIgnoreCmdApacheFakegooglebot(self):
+		if not STOCK: # pragma: no cover
+			raise unittest.SkipTest('Skip test because of no STOCK config')
+		cmd = os.path.join(STOCK_CONF_DIR, "filter.d/ignorecommands/apache-fakegooglebot")
+		## below test direct as python module:
+		mod = Utils.load_python_module(cmd)
+		self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "128.178.50.12"])))
+		self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "192.0.2.1"])))
+		bot_ips = ['66.249.66.1']
+		for ip in bot_ips:
+			self.assertTrue(mod.is_googlebot(mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip)
+		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd])))
+		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"])))
+		## via command:
+		self.filter.setIgnoreCommand(cmd + " <ip>")
+		for ip in bot_ips:
+			self.assertTrue(self.filter.inIgnoreIPList(str(ip)), "test of googlebot ip %s failed" % ip)
+			self.assertLogged('-- returned successfully')
+			self.pruneLog()
+		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
+		self.assertLogged('Argument must be a single valid IP.')
+		self.pruneLog()
+		self.filter.setIgnoreCommand(cmd + " bad arguments <ip>")
+		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
+		self.assertLogged('Please provide a single IP as an argument.')
+
+
 
 class LogFile(LogCaptureTestCase):
 

From 0ae932ba5e5e364e22392e1ecd2d72a75e427f62 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 15:29:56 +0200
Subject: [PATCH 211/623] setup fix for python3, bypass directories
 (__pycache__) created after ignore command was tested

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b3645f01..5754db49 100755
--- a/setup.py
+++ b/setup.py
@@ -205,7 +205,7 @@ setup(
 			glob("config/filter.d/*.conf")
 		),
 		('/etc/fail2ban/filter.d/ignorecommands',
-			glob("config/filter.d/ignorecommands/*")
+			[p for p in glob("config/filter.d/ignorecommands/*") if isfile(p)]
 		),
 		('/etc/fail2ban/action.d',
 			glob("config/action.d/*.conf") +

From 2d2d4cf1855f6edc5a3a4738d8789f9c27719e78 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 15:43:39 +0200
Subject: [PATCH 212/623] amend to c2d2e79b0d48bf66b04c3772c2419f30a4b1f9db:
 fixed sporadically bug in getBanListExtendedCymruInfo:     except
 dns.resolver.NXDOMAIN: UnboundLocalError: local variable 'dns' referenced
 before assignment

---
 fail2ban/server/banmanager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py
index 9865c2d3..1275d3a4 100644
--- a/fail2ban/server/banmanager.py
+++ b/fail2ban/server/banmanager.py
@@ -135,6 +135,7 @@ class BanManager:
 	def getBanListExtendedCymruInfo(self, timeout=10):
 		return_dict = {"asn": [], "country": [], "rir": []}
 		if not hasattr(self, 'dnsResolver'):
+			global dns
 			try:
 				import dns.exception
 				import dns.resolver

From 7e8575cc5609f4ebd8a2503bc79ebac078becddd Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 13 Oct 2016 17:09:49 +0200
Subject: [PATCH 213/623] tests of fail2ban-regex extended to cover
 exec_command_line also; Closes #1573

---
 fail2ban/client/fail2banregex.py        | 25 ++++++----------
 fail2ban/tests/fail2banregextestcase.py | 38 ++++++++++++++++++++++++-
 2 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index c05006f7..2d45c31d 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -136,13 +136,12 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   "\"systemd-journal\" only"),
 		Option('-l', "--log-level",
 			   dest="log_level",
-			   default=None,
+			   default='critical',
 			   help="Log level for the Fail2Ban logger to use"),
 		Option('-v', '--verbose', action="count", dest="verbose",
-			   default=None,
+			   default=0,
 			   help="Increase verbosity"),
 		Option("--verbosity", action="store", dest="verbose", type=int,
-			   default=None,
 			   help="Set numerical level of verbosity (0..4)"),
 		Option("-D", "--debuggex", action='store_true',
 			   help="Produce debuggex.com urls for debugging there"),
@@ -556,9 +555,9 @@ class Fail2banRegex(object):
 		return True
 
 
-def exec_command_line():
+def exec_command_line(*args):
 	parser = get_opt_parser()
-	(opts, args) = parser.parse_args()
+	(opts, args) = parser.parse_args(*args)
 	if opts.print_no_missed and opts.print_all_missed:
 		sys.stderr.write("ERROR: --print-no-missed and --print-all-missed are mutually exclusive.\n\n")
 		parser.print_help()
@@ -572,27 +571,21 @@ def exec_command_line():
 	if not len(args) in (2, 3):
 		sys.stderr.write("ERROR: provide both <LOG> and <REGEX>.\n\n")
 		parser.print_help()
-		return False
+		sys.exit(-1)
 
 	output( "" )
 	output( "Running tests" )
 	output( "=============" )
 	output( "" )
 
-	# TODO: taken from -testcases -- move common functionality somewhere
-	if opts.log_level is not None:
-		# so we had explicit settings
-		logSys.setLevel(str2LogLevel(opts.log_level))
-	else:
-		# suppress the logging but it would leave unittests' progress dots
-		# ticking, unless like with '-l critical' which would be silent
-		# unless error occurs
-		logSys.setLevel(logging.CRITICAL)
+	# Log level (default critical):
+	opts.log_level = str2LogLevel(opts.log_level)
+	logSys.setLevel(opts.log_level)
 
 	# Add the default logging handler
 	stdout = logging.StreamHandler(sys.stdout)
 
-	fmt = '%(levelname)-1.1s: %(message)s' if opts.verbose <= 1 else '%(message)s'
+	fmt = '%(levelname)-1.1s: %(message)s' if opts.verbose <= 1 else ' %(message)s'
 
 	if opts.log_traceback:
 		Formatter = FormatterWithTraceBack
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 0154e5af..a3d3cfb5 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -24,9 +24,10 @@ __copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Con
 __license__ = "GPL"
 
 import os
+import sys
 
 from ..client import fail2banregex
-from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output
+from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output
 from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase, logSys
 from .utils import CONFIG_DIR
 
@@ -39,12 +40,38 @@ fail2banregex.output = _test_output
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
+DEV_NULL = None
 
 def _Fail2banRegex(*args):
 	parser = get_opt_parser()
 	(opts, args) = parser.parse_args(list(args))
 	return (opts, args, Fail2banRegex(opts))
 
+class ExitException(Exception):
+	def __init__(self, code):
+		self.code = code
+		self.msg = 'Exit with code: %s' % code
+
+def _test_exec_command_line(*args):
+	def _exit(code=0):
+		raise ExitException(code)
+	global DEV_NULL
+	_org = {'exit': sys.exit, 'stdout': sys.stdout, 'stderr': sys.stderr}
+	_exit_code = 0
+	sys.exit = _exit
+	if not DEV_NULL: DEV_NULL = open(os.devnull, "w")
+	sys.stderr = sys.stdout = DEV_NULL
+	try:
+		exec_command_line(list(args))
+	except ExitException as e:
+		_exit_code = e.code
+	finally:
+		sys.exit = _org['exit']
+		sys.stdout = _org['stdout']
+		sys.stderr = _org['stderr']
+	return _exit_code
+
+
 class Fail2banRegexTest(LogCaptureTestCase):
 
 	RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>"
@@ -184,4 +211,13 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 		self.assertLogged('https://')
 
+	def testExecCmdLine_Usage(self):
+		self.assertNotEqual(_test_exec_command_line(), 0)
 
+	def testExecCmdLine_Direct(self):
+		self.assertEqual(_test_exec_command_line(
+			'-l', 'info',
+			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
+			r"Authentication failure for .*? from <HOST>$"
+		), 0)
+		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')

From 44f93bfbff0adbcf8e85b6f1b6be315d166cb063 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 13 Oct 2016 18:24:55 +0200
Subject: [PATCH 214/623] increase coverage, better test and output of errors

---
 fail2ban/client/fail2banregex.py        | 39 ++++++++++++-------------
 fail2ban/tests/fail2banregextestcase.py | 12 ++++++--
 2 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 2d45c31d..4096ac4e 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -40,7 +40,7 @@ from optparse import OptionParser, Option
 
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 
-try:
+try: # pragma: no cover
 	from systemd import journal
 	from ..server.filtersystemd import FilterSystemd
 except ImportError:
@@ -80,7 +80,7 @@ def pprint_list(l, header=None):
 		s = ''
 	output( s + "|  " + "\n|  ".join(l) + '\n`-' )
 
-def journal_lines_gen(myjournal):
+def journal_lines_gen(myjournal): # pragma: no cover
 	while True:
 		try:
 			entry = myjournal.get_next()
@@ -342,7 +342,7 @@ class Fail2banRegex(object):
 				found = True
 				regex = self._ignoreregex[ret].inc()
 		except RegexException as e:
-			output( e )
+			output( 'ERROR: %s' % e )
 			return False
 		return found
 
@@ -359,10 +359,7 @@ class Fail2banRegex(object):
 				regex.inc()
 				regex.appendIP(match)
 		except RegexException as e:
-			output( e )
-			return False
-		except IndexError:
-			output( "Sorry, but no <HOST> found in regex" )
+			output( 'ERROR: %s' % e )
 			return False
 		for bufLine in orgLineBuffer[int(fullBuffer):]:
 			if bufLine not in self._filter._Filter__lineBuffer:
@@ -508,10 +505,13 @@ class Fail2banRegex(object):
 
 		cmd_log, cmd_regex = args[:2]
 
-		if not self.readRegex(cmd_regex, 'fail'):
-			return False
-
-		if len(args) == 3 and not self.readRegex(args[2], 'ignore'):
+		try:
+			if not self.readRegex(cmd_regex, 'fail'):
+				return False
+			if len(args) == 3 and not self.readRegex(args[2], 'ignore'):
+				return False
+		except RegexException as e:
+			output( 'ERROR: %s' % e )
 			return False
 
 		if os.path.isfile(cmd_log):
@@ -558,18 +558,17 @@ class Fail2banRegex(object):
 def exec_command_line(*args):
 	parser = get_opt_parser()
 	(opts, args) = parser.parse_args(*args)
-	if opts.print_no_missed and opts.print_all_missed:
-		sys.stderr.write("ERROR: --print-no-missed and --print-all-missed are mutually exclusive.\n\n")
-		parser.print_help()
-		sys.exit(-1)
-	if opts.print_no_ignored and opts.print_all_ignored:
-		sys.stderr.write("ERROR: --print-no-ignored and --print-all-ignored are mutually exclusive.\n\n")
-		parser.print_help()
-		sys.exit(-1)
+	errors = []
+	if opts.print_no_missed and opts.print_all_missed: # pragma: no cover
+		errors.append("ERROR: --print-no-missed and --print-all-missed are mutually exclusive.")
+	if opts.print_no_ignored and opts.print_all_ignored: # pragma: no cover
+		errors.append("ERROR: --print-no-ignored and --print-all-ignored are mutually exclusive.")
 
 	# We need 2 or 3 parameters
 	if not len(args) in (2, 3):
-		sys.stderr.write("ERROR: provide both <LOG> and <REGEX>.\n\n")
+		errors.append("ERROR: provide both <LOG> and <REGEX>.")
+	if errors:
+		sys.stderr.write("\n".join(errors) + "\n\n")
 		parser.print_help()
 		sys.exit(-1)
 
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index a3d3cfb5..4445fe6b 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -96,14 +96,14 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"test", r".** from <HOST>$"
 		)
-		self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args))
+		self.assertFalse(fail2banRegex.start(opts, args))
 		self.assertLogged("Unable to compile regular expression")
 
 	def testWrongIngnoreRE(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"test", r".*? from <HOST>$", r".**"
 		)
-		self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args))
+		self.assertFalse(fail2banRegex.start(opts, args))
 		self.assertLogged("Unable to compile regular expression")
 
 	def testDirectFound(self):
@@ -221,3 +221,11 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			r"Authentication failure for .*? from <HOST>$"
 		), 0)
 		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
+		
+	def testExecCmdLine_MissFailID(self):
+		self.assertNotEqual(_test_exec_command_line(
+			'-l', 'info',
+			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
+			r"Authentication failure"
+		), 0)
+		self.assertLogged('No failure-id group in ')

From 98f87a1a5273047eff67185d8916a4b72987ac2f Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Fri, 14 Oct 2016 22:32:02 +0200
Subject: [PATCH 215/623] better server-ready event: notify waiting thread if
 server really ready (communication ready) or failed to start

---
 fail2ban/client/fail2banserver.py | 10 ++++++++--
 fail2ban/server/asyncserver.py    |  4 ++++
 fail2ban/server/server.py         |  1 +
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index aba8f8ca..dfee34d2 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -184,13 +184,19 @@ class Fail2banServer(Fail2banCmdLine):
 				logSys.log(5, '  server phase %s', phase)
 				if not phase.get('start', False):
 					raise ServerExecutionException('Async configuration of server failed')
+				# event for server ready flag:
+				def _server_ready():
+					phase['start-ready'] = True
+					logSys.log(5, '  server phase %s', phase)
+				# notify waiting thread if server really ready
+				self._conf['onstart'] = _server_ready
 
 			# Start server, daemonize it, etc.
 			pid = os.getpid()
 			server = Fail2banServer.startServerDirect(self._conf, background)
+			# notify waiting thread server ready resp. done (background execution, error case, etc):
 			if not async:
-				phase['start-ready'] = True
-				logSys.log(5, '  server phase %s', phase)
+				_server_ready()
 			# If forked - just exit other processes
 			if pid != os.getpid(): # pragma: no cover
 				os._exit(0)
diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index a489b585..d1818d7a 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -147,6 +147,7 @@ class AsyncServer(asyncore.dispatcher):
 		self.__sock = "/var/run/fail2ban/fail2ban.sock"
 		self.__init = False
 		self.__active = False
+		self.onstart = None
 
 	##
 	# Returns False as we only read the socket first.
@@ -196,6 +197,9 @@ class AsyncServer(asyncore.dispatcher):
 		self.listen(1)
 		# Sets the init flag.
 		self.__init = self.__loop = self.__active = True
+		# Execute on start event (server ready):
+		if self.onstart:
+			self.onstart()
 		# Event loop as long as active:
 		loop(lambda: self.__loop, use_poll=use_poll)
 		self.__active = False
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 501083ed..e43f56ee 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -148,6 +148,7 @@ class Server:
 		logSys.debug("Starting communication")
 		try:
 			self.__asyncServer = AsyncServer(self.__transm)
+			self.__asyncServer.onstart = conf.get('onstart')
 			self.__asyncServer.start(sock, force)
 		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)

From 1ef367e77a5d7f38baf2b2f186504ad3228a0230 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Fri, 14 Oct 2016 20:55:15 +0200
Subject: [PATCH 216/623] fixes log format by starting server with
 `--loglevel=debug` without specifying of verbosity level

---
 fail2ban/server/server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index e43f56ee..8c001ea3 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -599,6 +599,7 @@ class Server:
 			if logger.getEffectiveLevel() <= logging.DEBUG: # pragma: no cover
 				if self.__verbose is None:
 					self.__verbose = logging.DEBUG - logger.getEffectiveLevel() + 1
+			if self.__verbose is not None and self.__verbose > 2: # pragma: no cover
 				fmt = getVerbosityFormat(self.__verbose-1)
 			# tell the handler to use this format
 			hdlr.setFormatter(logging.Formatter(fmt))

From 57a77952820c312ab362e19cd83542cdbec55a95 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Fri, 14 Oct 2016 23:57:35 +0200
Subject: [PATCH 217/623] code coverage of server module: multiple ignoreregex

---
 fail2ban/tests/config/filter.d/zzz-generic-example.conf | 6 +++++-
 fail2ban/tests/files/logs/zzz-generic-example           | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
index e2ae91b0..df30d725 100644
--- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf
+++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
@@ -15,4 +15,8 @@ before = common.conf
 _daemon = test-demo
 
 failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
-ignoreregex = 
+            ^%(__prefix_line)sF2B: error from <HOST>$
+
+# just to test multiple ignoreregex:
+ignoreregex = ^%(__prefix_line)sF2B: error from 192.0.2.251$
+              ^%(__prefix_line)sF2B: error from 192.0.2.252$
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index 2a4fb789..2044c387 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -45,3 +45,10 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
 # -- wrong time direct in journal-line (used last known date):
 # failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2
+
+# failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" }
+[Jun 21 16:56:02] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.250
+# failJSON: { "match": false, "desc": "test 1st ignoreregex" }
+[Jun 21 16:56:03] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.251
+# failJSON: { "match": false, "desc": "test 2nd ignoreregex" }
+[Jun 21 16:56:04] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.252

From 5b403090526495765dcc2250767b6c8a18d89bdd Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Fri, 14 Oct 2016 23:57:52 +0200
Subject: [PATCH 218/623] code coverage of server module: switch backend

---
 fail2ban/tests/fail2banclienttestcase.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index d8fc156b..66ff61b3 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -735,7 +735,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
 				_out_file(fn)
 
-		def _write_jail_cfg(enabled=(1, 2), actions=()):
+		def _write_jail_cfg(enabled=(1, 2), actions=(), backend="polling"):
 			_write_file(pjoin(cfg, "jail.conf"), "w",
 				"[INCLUDES]", "",
 				"[DEFAULT]", "",
@@ -744,7 +744,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",
 				"",
-				"[test-jail1]", "backend = polling", "filter =", 
+				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",
 				"         test-action1[name='%(__name__)s']" if 1 in actions else "",
 				"         test-action2[name='%(__name__)s']" if 2 in actions else "",
@@ -755,7 +755,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"            ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
 				"enabled = true" if 1 in enabled else "",
 				"",
-				"[test-jail2]", "backend = polling", "filter =", 
+				"[test-jail2]", "backend = " + backend, "filter =", 
 				"action =",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
@@ -999,6 +999,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail1] Ban 192.0.2.4", all=True
 		)
 
+		# backend-switch (restart instead of reload):
+		self.pruneLog("[test-phase 8a]")
+		_write_jail_cfg(enabled=[1], backend="xxx-unknown-backend-zzz")
+		self.execFailed(startparams, "reload")
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
+		self.assertLogged(
+			"Restart jail 'test-jail1' (reason: 'polling' != ", 
+			"Unknown backend ", all=True)
+
+		self.pruneLog("[test-phase 8b]")
+		_write_jail_cfg(enabled=[1])
+		self.execSuccess(startparams, "reload")
+		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)	
+
 		# several small cases (cover several parts):
 		self.pruneLog("[test-phase end-1]")
 		# wrong jail (not-started):

From 7f8c48d59e11ca9f7912a819302cb203408d6714 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 15 Oct 2016 00:39:25 +0200
Subject: [PATCH 219/623] code review (e. g. remove code duplication) and
 coverage

---
 fail2ban/client/fail2banclient.py        |  2 +-
 fail2ban/server/server.py                | 38 ++++++++++--------------
 fail2ban/tests/fail2banclienttestcase.py |  9 +++++-
 3 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 331c5f34..007afd57 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -310,7 +310,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 					# stop options - jail name or --all
 					break
 			if self.__ping(timeout=-1):
-				if len(cmd) == 1:
+				if len(cmd) == 1 or cmd[1] == '--all':
 					jail = '--all'
 					ret, stream = self.readConfig()
 				else:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 8c001ea3..efd47b09 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -81,12 +81,12 @@ class Server:
 		}
 		self.__prev_signals = {}
 
-	def __sigTERMhandler(self, signum, frame):
-		logSys.debug("Caught signal %d. Exiting" % signum)
+	def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested
+		logSys.debug("Caught signal %d. Exiting", signum)
 		self.quit()
 	
-	def __sigUSR1handler(self, signum, fname):
-		logSys.debug("Caught signal %d. Flushing logs" % signum)
+	def __sigUSR1handler(self, signum, fname): # pragma: no cover - indirect tested
+		logSys.debug("Caught signal %d. Flushing logs", signum)
 		self.flushLogs()
 
 	def _rebindSignal(self, s, new):
@@ -137,12 +137,12 @@ class Server:
 
 		# Creates a PID file.
 		try:
-			logSys.debug("Creating PID file %s" % pidfile)
+			logSys.debug("Creating PID file %s", pidfile)
 			pidFile = open(pidfile, 'w')
 			pidFile.write("%s\n" % os.getpid())
 			pidFile.close()
-		except IOError as e:
-			logSys.error("Unable to create PID file: %s" % e)
+		except (OSError, IOError) as e: # pragma: no cover
+			logSys.error("Unable to create PID file: %s", e)
 		
 		# Start the communication
 		logSys.debug("Starting communication")
@@ -154,10 +154,10 @@ class Server:
 			logSys.error("Could not start server: %s", e)
 		# Removes the PID file.
 		try:
-			logSys.debug("Remove PID file %s" % pidfile)
+			logSys.debug("Remove PID file %s", pidfile)
 			os.remove(pidfile)
-		except OSError as e:
-			logSys.error("Unable to remove PID file: %s" % e)
+		except (OSError, IOError) as e: # pragma: no cover
+			logSys.error("Unable to remove PID file: %s", e)
 		logSys.info("Exiting Fail2ban")
 	
 	def quit(self):
@@ -250,7 +250,7 @@ class Server:
 	def reloadJails(self, name, opts, begin):
 		if begin:
 			# begin reload:
-			if self.__reload_state and (name == '--all' or self.__reload_state.get(name)):
+			if self.__reload_state and (name == '--all' or self.__reload_state.get(name)): # pragma: no cover
 				raise ValueError('Reload already in progress')
 			logSys.info("Reload " + (("jail %s" % name) if name != '--all' else "all jails"))
 			with self.__lock:
@@ -381,11 +381,8 @@ class Server:
 
 	def addFailRegex(self, name, value, multiple=False):
 		flt = self.__jails[name].filter
-		if multiple:
-			for value in value:
-				logSys.debug("  failregex: %r", value)
-				flt.addFailRegex(value)
-		else:
+		if not multiple: value = (value,)
+		for value in value:
 			logSys.debug("  failregex: %r", value)
 			flt.addFailRegex(value)
 	
@@ -397,11 +394,8 @@ class Server:
 	
 	def addIgnoreRegex(self, name, value, multiple=False):
 		flt = self.__jails[name].filter
-		if multiple:
-			for value in value:
-				logSys.debug("  ignoreregex: %r", value)
-				flt.addIgnoreRegex(value)
-		else:
+		if not multiple: value = (value,)
+		for value in value:
 			logSys.debug("  ignoreregex: %r", value)
 			flt.addIgnoreRegex(value)
 	
@@ -671,7 +665,7 @@ class Server:
 			if Fail2BanDb is not None:
 				self.__db = Fail2BanDb(filename)
 				self.__db.delAllJails()
-			else:
+			else: # pragma: no cover
 				logSys.error(
 					"Unable to import fail2ban database module as sqlite "
 					"is not available.")
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 66ff61b3..e212ff23 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -1026,4 +1026,11 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertNotLogged(
 			"Creating new jail 'test-jail2'",
 			"Jail 'test-jail2' started", all=True)
-		self.pruneLog()
+
+		# restart all jails (without restart server):
+		self.pruneLog("[test-phase end-2]")
+		self.execSuccess(startparams,
+			"--async", "reload", "--restart", "--all")
+		self.assertLogged(
+			"Jail 'test-jail1' stopped", 
+			"Jail 'test-jail1' started", all=True)

From 10bdadaef2756629ac370d5e0c1374e98b7fc658 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 15 Oct 2016 19:00:01 +0200
Subject: [PATCH 220/623] fixed sporadically (multi-threading) errors by
 reload/stop/start of polling filter inside getModified (so prevents to stop
 running main cycle)

---
 fail2ban/server/filterpoll.py | 62 ++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 59310bed..e785a0e4 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -97,32 +97,40 @@ class FilterPoll(FileFilter):
 
 	def run(self):
 		while self.active:
-			if logSys.getEffectiveLevel() <= 6:
-				logSys.log(6, "Woke up idle=%s with %d files monitored",
-						   self.idle, self.getLogCount())
-			if self.idle:
-				if not Utils.wait_for(lambda: not self.active or not self.idle, 
-					self.sleeptime * 10, self.sleeptime
-				):
-					self.ticks += 1
-					continue
-			# Get file modification
-			modlst = []
-			Utils.wait_for(lambda: not self.active or self.getModified(modlst),
-				self.sleeptime)
-			for filename in modlst:
-				self.getFailures(filename)
-				self.__modified = True
+			try:
+				if logSys.getEffectiveLevel() <= 6:
+					logSys.log(6, "Woke up idle=%s with %d files monitored",
+							   self.idle, self.getLogCount())
+				if self.idle:
+					if not Utils.wait_for(lambda: not self.active or not self.idle, 
+						self.sleeptime * 10, self.sleeptime
+					):
+						self.ticks += 1
+						continue
+				# Get file modification
+				modlst = []
+				Utils.wait_for(lambda: not self.active or self.getModified(modlst),
+					self.sleeptime)
+				for filename in modlst:
+					self.getFailures(filename)
+					self.__modified = True
 
-			self.ticks += 1
-			if self.__modified:
-				try:
-					while True:
-						ticket = self.failManager.toBan()
-						self.jail.putFailTicket(ticket)
-				except FailManagerEmpty:
-					self.failManager.cleanup(MyTime.time())
-				self.__modified = False
+				self.ticks += 1
+				if self.__modified:
+					try:
+						while True:
+							ticket = self.failManager.toBan()
+							self.jail.putFailTicket(ticket)
+					except FailManagerEmpty:
+						self.failManager.cleanup(MyTime.time())
+					self.__modified = False
+			except Exception as e: # pragma: no cover
+				if not self.active: # if not active - error by stop...
+					break
+				logSys.error("Caught unhandled exception in main cycle: %r", e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+				# incr common error counter:
+				self.commonError()
 		logSys.debug("[%s] filter terminated", self.jailName)
 		return True
 
@@ -151,9 +159,9 @@ class FilterPoll(FileFilter):
 			return True
 		except Exception as e:
 			# stil alive (may be deleted because multi-threaded):
-			if not self.getLog(filename):
+			if not self.getLog(filename) or self.__prevStats.get(filename) is None:
 				logSys.warning("Log %r seems to be down: %s", filename, e)
-				return
+				return False
 			# log error:
 			if self.__file404Cnt[filename] < 2:
 				logSys.error("Unable to get stat on %s because of: %s",

From a7d9de8c5264b20f1fa63071f3f4eeed8735b54a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 27 Sep 2016 17:53:45 +0200
Subject: [PATCH 221/623] [temp commit] 1st try to optimize
 datedetector/datetemplate functionality (fix ambiguous resp. misleading date
 detection if several formats used in log resp. by format switch after restart
 of some services): * Misleading date patterns defined more precisely (using
 extended syntax %E[mdHMS]   for exact two-digit match) *
 `filter.d/freeswitch.conf`     - Optional prefixes (server, daemon, dual
 time) if systemd daemon logs used (gh-1548)     - User part rewritten to
 accept IPv6 resp. domain after "@" (gh-1548)

---
 config/filter.d/freeswitch.conf        |  15 +++-
 fail2ban/server/datedetector.py        |  99 +++++++++++++++-------
 fail2ban/server/datetemplate.py        |  32 +++----
 fail2ban/server/strptime.py            |  16 ++++
 fail2ban/tests/datedetectortestcase.py | 112 ++++++++++++++-----------
 fail2ban/tests/files/logs/freeswitch   |   5 ++
 6 files changed, 181 insertions(+), 98 deletions(-)

diff --git a/config/filter.d/freeswitch.conf b/config/filter.d/freeswitch.conf
index 1ef5a256..29940240 100644
--- a/config/filter.d/freeswitch.conf
+++ b/config/filter.d/freeswitch.conf
@@ -8,10 +8,21 @@
 # IP addresses on your LAN.
 #
 
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
 [Definition]
 
-failregex = ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[.*\] from ip <HOST>$
-            ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ Can't find user \[\d+@\d+\.\d+\.\d+\.\d+\] from <HOST>$
+_daemon = freeswitch
+
+# Prefix contains common prefix line (server, daemon, etc.) and 2 datetimes if used systemd backend
+_pref_line = ^%(__prefix_line)s(?:\d+-\d+-\d+ \d+:\d+:\d+\.\d+)?
+
+failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip <HOST>$
+            %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ Can't find user \[[^@]+@[^\]]+\] from <HOST>$
 
 ignoreregex =
 
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 5281fc59..b1f97af7 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -21,6 +21,7 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
+import copy
 import time
 
 from threading import Lock
@@ -35,6 +36,8 @@ logLevel = 6
 
 
 class DateDetectorCache(object):
+	"""Implements the caching of the default templates list.
+	"""
 	def __init__(self):
 		self.__lock = Lock()
 		self.__templates = list()
@@ -51,26 +54,44 @@ class DateDetectorCache(object):
 
 	def _cacheTemplate(self, template):
 		"""Cache Fail2Ban's default template.
+
 		"""
 		if isinstance(template, str):
+			# exact given template with word benin-end boundary:
 			template = DatePatternRegex(template)
+		# additional template, that prefers datetime at start of a line (safety+performance feature):
+		template2 = copy.copy(template)
+		if hasattr(template, 'pattern'):
+			regex = template.pattern
+			wordEnd = True
+		else:
+			regex = template.regex
+			wordEnd = False
+		template2.setRegex(regex, wordBegin='start', wordEnd=wordEnd)
+		if template2.name != template.name:
+			self.__templates.append(template2)
+		# add template:
 		self.__templates.append(template)
 
 	def _addDefaultTemplate(self):
 		"""Add resp. cache Fail2Ban's default set of date templates.
 		"""
+		# ISO 8601, simple date, optional subsecond and timezone:
+		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59
+		# simple date: 2005/01/23 21:59:59 
+		# custom for syslog-ng 2006.12.21 06:43:20
+		self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
+		# 20050123T215959, 20050123 215959
+		self._cacheTemplate("%Y%Em%Ed[T ]%EH%EM%ES(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
+		# prefixed with optional time zone (monit):
+		# PDT Apr 16 21:05:29
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
 		self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?")
-		# simple date, optional subsecond (proftpd):
-		# 2005-01-23 21:59:59 
-		# simple date: 2005/01/23 21:59:59 
-		# custom for syslog-ng 2006.12.21 06:43:20
-		self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?")
 		# simple date too (from x11vnc): 23/01/2005 21:59:59 
 		# and with optional year given by 2 digits: 23/01/05 21:59:59 
 		# (See http://bugs.debian.org/537610)
@@ -79,37 +100,48 @@ class DateDetectorCache(object):
 		# Apache format optional time zone:
 		# [31/Oct/2006:09:22:55 -0000]
 		# 26-Jul-2007 15:20:52
+		# named 26-Jul-2007 15:20:52.252
+		# roundcube 26-Jul-2007 15:20:52 +0200
 		self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
 		# CPanel 05/20/2008:01:57:39
 		self._cacheTemplate("%m/%d/%Y:%H:%M:%S")
-		# named 26-Jul-2007 15:20:52.252 
-		# roundcube 26-Jul-2007 15:20:52 +0200
 		# 01-27-2012 16:22:44.252
 		# subseconds explicit to avoid possible %m<->%d confusion
-		# with previous
-		self._cacheTemplate("%m-%d-%Y %H:%M:%S\.%f")
+		# with previous ("%d-%m-%Y %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S")
+		self._cacheTemplate("%m-%d-%Y %H:%M:%S(?:\.%f)?")
 		# TAI64N
-		template = DateTai64n()
-		template.name = "TAI64N"
-		self._cacheTemplate(template)
+		self._cacheTemplate(DateTai64n())
 		# Epoch
-		template = DateEpoch()
-		template.name = "Epoch"
-		self._cacheTemplate(template)
-		# ISO 8601
-		self._cacheTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?")
+		self._cacheTemplate(DateEpoch())
 		# Only time information in the log
 		self._cacheTemplate("^%H:%M:%S")
 		# <09/16/08@05:03:30>
 		self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>")
 		# MySQL: 130322 11:46:11
-		self._cacheTemplate("^%y%m%d  ?%H:%M:%S")
+		self._cacheTemplate("%y%Em%Ed  ?%H:%M:%S")
 		# Apache Tomcat
 		self._cacheTemplate("%b %d, %Y %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
 		self._cacheTemplate("^%b-%d-%y %H:%M:%S")
 
 
+class DateDetectorTemplate(object):
+	"""Used for "shallow copy" of the template object.
+
+	Prevents collectively usage of hits/lastUsed in cached templates
+	"""
+	__slots__ = ('template', 'hits', 'lastUsed')
+	def __init__(self, template):
+		self.template = template
+		self.hits = 0
+		self.lastUsed = 0
+
+	def __getattr__(self, name):
+		""" Returns attribute of template (called for parameters not in slots)
+		"""
+		return getattr(self.template, name)
+
+
 class DateDetector(object):
 	"""Manages one or more date templates to find a date within a log line.
 
@@ -132,7 +164,7 @@ class DateDetector(object):
 			raise ValueError(
 				"There is already a template with name %s" % name)
 		self.__known_names.add(name)
-		self.__templates.append(template)
+		self.__templates.append(DateDetectorTemplate(template))
 
 	def appendTemplate(self, template):
 		"""Add a date template to manage and use in search of dates.
@@ -186,13 +218,14 @@ class DateDetector(object):
 		"""
 		i = 0
 		with self.__lock:
-			for template in self.__templates:
+			for ddtemplate in self.__templates:
+				template = ddtemplate.template
 				match = template.matchDate(line)
 				if not match is None:
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "Matched time template %s", template.name)
-					template.hits += 1
-					template.lastUsed = time.time()
+					ddtemplate.hits += 1
+					ddtemplate.lastUsed = time.time()
 					# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
 					if i:
 						self._reorderTemplate(i)
@@ -234,7 +267,8 @@ class DateDetector(object):
 				except ValueError:
 					return None
 		with self.__lock:
-			for template in self.__templates:
+			for ddtemplate in self.__templates:
+				template = ddtemplate.template
 				try:
 					date = template.getDate(line)
 					if date is None:
@@ -261,14 +295,21 @@ class DateDetector(object):
 		  ## current hits and time the template was long unused:
 			untime = template.lastUsed - self.__unusedTime
 			hits = template.hits
+			## try to move faster (first 2 if it still unused, or half of part to current template position):
+			phits = 0
+			for pos in (0, 1, num // 2):
+				phits = templates[pos].hits
+				if not phits:
+					break
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
-			if hits > templates[num-1].hits + 5 or templates[num-1].lastUsed < untime:
-				## try to move faster (half of part to current template):
-				pos = num // 2
-				## if not larger - move slow (exact 1 position):
-				if hits <= templates[pos].hits or templates[pos].lastUsed < untime:
+			if not phits or hits > phits + 5 or templates[pos].lastUsed < untime:
+				## if not larger (and target position recently used) - move slow (exact 1 position):
+				if hits <= phits and templates[pos].lastUsed > untime:
 					pos = num-1
+					## if still smaller and template at position used, don't move:
+					if hits < templates[pos].hits and templates[pos].lastUsed > untime:
+						return
 				templates[pos], templates[num] = template, templates[pos]
 
 
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 4f609e78..5fcc16a4 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -27,7 +27,7 @@ __license__ = "GPL"
 import re
 from abc import abstractmethod
 
-from .strptime import reGroupDictStrptime, timeRE
+from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE
 from ..helpers import getLogger
 
 logSys = getLogger(__name__)
@@ -49,8 +49,6 @@ class DateTemplate(object):
 		self._name = ""
 		self._regex = ""
 		self._cRegex = None
-		self.hits = 0
-		self.lastUsed = 0
 
 	@property
 	def name(self):
@@ -88,9 +86,11 @@ class DateTemplate(object):
 		"""
 		regex = regex.strip()
 		if wordBegin and not re.search(r'^\^', regex):
-			regex = r'(?=^|\b|\W)' + regex
+			regex = (r'(?<=^|\b)' if wordBegin != 'start' else r"^(?<=\W)?") + regex
+			self._name = ('[*WD-BEG]' if wordBegin != 'start' else '[^LN-BEG]') + self._name
 		if wordEnd and not re.search(r'\$$', regex):
 			regex += r'(?=\b|\W|$)'
+			self._name += ('[*WD-END]' if wordEnd else '')
 		self._regex = regex
 
 	regex = property(getRegex, setRegex, doc=
@@ -140,7 +140,9 @@ class DateEpoch(DateTemplate):
 
 	def __init__(self):
 		DateTemplate.__init__(self)
-		self.regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=audit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
+		self.name = "Epoch"
+		self.setRegex(r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))",
+			wordBegin=False) ;# already line begin resp. word begin anchored
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
@@ -178,7 +180,8 @@ class DatePatternRegex(DateTemplate):
 	regex
 	pattern
 	"""
-	_patternRE = re.compile(r"%%(%%|[%s])" % "".join(timeRE.keys()))
+	
+	_patternRE = re.compile(getTimePatternRE())
 	_patternName = {
 		'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
 		'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
@@ -188,11 +191,11 @@ class DatePatternRegex(DateTemplate):
 	for _key in set(timeRE) - set(_patternName): # may not have them all...
 		_patternName[_key] = "%%%s" % _key
 
-	def __init__(self, pattern=None):
+	def __init__(self, pattern=None, **kwargs):
 		super(DatePatternRegex, self).__init__()
 		self._pattern = None
 		if pattern is not None:
-			self.pattern = pattern
+			self.setRegex(pattern, **kwargs)
 
 	@property
 	def pattern(self):
@@ -208,17 +211,13 @@ class DatePatternRegex(DateTemplate):
 
 	@pattern.setter
 	def pattern(self, pattern):
+		self.setRegex(pattern)
+
+	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
 		self._pattern = pattern
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
 		self._name = fmt % self._patternName
-		super(DatePatternRegex, self).setRegex(fmt % timeRE)
-
-	def setRegex(self, value):
-		raise NotImplementedError("Regex derived from pattern")
-
-	@DateTemplate.name.setter
-	def name(self, value):
-		raise NotImplementedError("Name derived from pattern")
+		super(DatePatternRegex, self).setRegex(fmt % timeRE, wordBegin, wordEnd)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
@@ -258,6 +257,7 @@ class DateTai64n(DateTemplate):
 
 	def __init__(self):
 		DateTemplate.__init__(self)
+		self.name = "TAI64N"
 		# We already know the format for TAI64N
 		# yoh: we should not add an additional front anchor
 		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 2e3c051c..c6a1f37e 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -26,8 +26,24 @@ from .mytime import MyTime
 
 locale_time = LocaleTime()
 timeRE = TimeRE()
+#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
+#timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
 timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
 
+# Extend build-in TimeRE with some exact (two-digit) patterns:
+timeRE['Ed'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
+timeRE['Em'] = r"(?P<m>1[0-2]|0[1-9])"
+timeRE['EH'] = r"(?P<H>2[0-3]|[0-1]\d)"
+timeRE['EM'] = r"(?P<M>[0-5]\d)"
+timeRE['ES'] = r"(?P<S>6[0-1]|[0-5]\d)"
+
+def getTimePatternRE():
+	keys = timeRE.keys()
+	return (r"%%(%%|%s|[%s])" % (
+		"|".join([k for k in keys if len(k) > 1]),
+		"".join([k for k in keys if len(k) == 1]),
+	))
+
 
 def reGroupDictStrptime(found_dict):
 	"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 0c3c306d..013e1e85 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -89,69 +89,79 @@ class DateDetectorTest(LogCaptureTestCase):
 		"""
 		dateUnix = 1106513999.0
 
-		for anchored, sdate in (
-			(False, "Jan 23 21:59:59"),
-			(False, "Sun Jan 23 21:59:59 2005"),
-			(False, "Sun Jan 23 21:59:59"),
-			(False, "Sun Jan 23 2005 21:59:59"),
-			(False, "2005/01/23 21:59:59"),
-			(False, "2005.01.23 21:59:59"),
-			(False, "23/01/2005 21:59:59"),
-			(False, "23/01/05 21:59:59"),
-			(False, "23/Jan/2005:21:59:59"),
-			(False, "23/Jan/2005:21:59:59 +0100"),
-			(False, "01/23/2005:21:59:59"),
-			(False, "2005-01-23 21:59:59"),
-		    (False, "2005-01-23 21:59:59,000"),	  # proftpd
-			(False, "23-Jan-2005 21:59:59"),
-			(False, "23-Jan-2005 21:59:59.02"),
-			(False, "23-Jan-2005 21:59:59 +0100"),
-			(False, "23-01-2005 21:59:59"),
-			(True, "1106513999"), # Portsetry
-			(False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break
-			(False, "@4000000041f4104f00000000"), # TAI64N
-			(False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC)
-			(False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ
-			(False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local
-			(True,  "<01/23/05@21:59:59>"),
-			(True,  "050123 21:59:59"), # MySQL
-			(True,  "Jan-23-05 21:59:59"), # ASSP like
-			(False, "Jan 23, 2005 9:59:59 PM"), # Apache Tomcat
-			(True,  "1106513999"), # Regular epoch
-			(True,  "1106513999.000"), # Regular epoch with millisec
-			(False, "audit(1106513999.000:987)"), # SELinux
+		for anchored, bound, sdate, rdate in (
+			(False, True,  "Jan 23 21:59:59", None),
+			(False, False, "Sun Jan 23 21:59:59 2005", None),
+			(False, False, "Sun Jan 23 21:59:59", None),
+			(False, False, "Sun Jan 23 2005 21:59:59", None),
+			(False, True,  "2005/01/23 21:59:59", None),
+			(False, True,  "2005.01.23 21:59:59", None),
+			(False, True,  "23/01/2005 21:59:59", None),
+			(False, True,  "23/01/05 21:59:59", None),
+			(False, True,  "23/Jan/2005:21:59:59", None),
+			(False, True,  "23/Jan/2005:21:59:59 +0100", None),
+			(False, True,  "01/23/2005:21:59:59", None),
+			(False, True,  "2005-01-23 21:59:59", None),
+			(False, True,  "2005-01-23 21:59:59,000", None),	  # proftpd
+			(False, True,  "23-Jan-2005 21:59:59", None),
+			(False, True,  "23-Jan-2005 21:59:59.02", None),
+			(False, True,  "23-Jan-2005 21:59:59 +0100", None),
+			(False, True,  "23-01-2005 21:59:59", None),
+			(True,  True,  "1106513999", None), # Portsetry
+			(False, True,  "01-23-2005 21:59:59.252", None), # reported on f2b, causes Feb29 fix to break
+			(False, False, "@4000000041f4104f00000000", None), # TAI64N
+			(False, True,  "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC)
+			(False, True,  "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ
+			(False, True,  "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local
+			(False, True,  "20050123T215959", None),   #Short ISO
+			(False, True,  "20050123 215959", None),   #Short ISO
+			(True,  True,  "<01/23/05@21:59:59>", None),
+			(False, True,  "050123 21:59:59", None), # MySQL
+			(True,  True,  "Jan-23-05 21:59:59", None), # ASSP like
+			(False, True,  "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat
+			(True,  True,  "1106513999", None), # Regular epoch
+			(True,  True,  "1106513999.000", None), # Regular epoch with millisec
+			(True,  True,  "[1106513999.000]", "1106513999.000"), # epoch squared
+			(False, True,  "audit(1106513999.000:987)", "1106513999.000"), # SELinux
+		):
+			logSys.debug('== test %r', (anchored, bound, sdate))
+			for should_match, prefix in (
+				(True,         ""),
+				(not anchored, "bogus-prefix "),
+				(False,        "word-boundary")
 			):
-			for should_match, prefix in ((True,     ""),
-										 (not anchored, "bogus-prefix ")):
+				if rdate is None: rdate = sdate
 				log = prefix + sdate + "[sshd] error: PAM: Authentication failure"
-
+				# if not allowed boundary test:
+				if not bound and prefix == "word-boundary": continue
+				logSys.debug('  -- test %-5s for %r', should_match, log)
 				# with getTime:
 				logtime = self.__datedetector.getTime(log)
 				if should_match:
-					self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log))
+					self.assertNotEqual(logtime, None,
+						"getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log))
 					( logUnix, logMatch ) = logtime
-					self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix))
-					if sdate.startswith('audit('):
-						# yes, special case, the group only matches the number
-						self.assertEqual(logMatch.group(), '1106513999.000')
-					else:
-						self.assertEqual(logMatch.group(), sdate)
+					self.assertEqual(logUnix, dateUnix,
+						"getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
+					self.assertEqual(logMatch.group(), rdate)
 				else:
-					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
+					self.assertEqual(logtime, None,
+						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
 				# with getTime(matchTime) - this combination used in filter:
-				matchTime = self.__datedetector.matchTime(log)
+				(timeMatch, template) = matchTime = self.__datedetector.matchTime(log)
 				logtime = self.__datedetector.getTime(log, matchTime)
+				logSys.debug('  -- found - %r', template.name if timeMatch else False)
 				if should_match:
-					self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s, anchored: %r, log: %s" % ( sdate, anchored, log))
+					self.assertNotEqual(logtime, None,
+						"getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log))
 					( logUnix, logMatch ) = logtime
-					self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix))
-					if sdate.startswith('audit('):
-						# yes, special case, the group only matches the number
-						self.assertEqual(logMatch.group(), '1106513999.000')
-					else:
-						self.assertEqual(logMatch.group(), sdate)
+					self.assertEqual(logUnix, dateUnix,
+						"getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
+					self.assertEqual(logMatch.group(), rdate)
 				else:
-					self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
+					self.assertEqual(logtime, None,
+						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
+				logSys.debug('  -- OK')
 
 	def testAllUniqueTemplateNames(self):
 		self.assertRaises(ValueError, self.__datedetector.appendTemplate,
diff --git a/fail2ban/tests/files/logs/freeswitch b/fail2ban/tests/files/logs/freeswitch
index f151995c..2579e6f4 100644
--- a/fail2ban/tests/files/logs/freeswitch
+++ b/fail2ban/tests/files/logs/freeswitch
@@ -9,3 +9,8 @@
 2013-12-31 17:39:54.767815 [WARNING] sofia_reg.c:2531 Can't find user [1001@192.168.2.51] from 5.11.47.236
 # failJSON: { "time": "2013-12-31T17:39:54", "match": true, "host": "185.24.234.141" }
 2013-12-31 17:39:54.767815 [WARNING] sofia_reg.c:2531 Can't find user [100@192.168.2.51] from 185.24.234.141
+
+# failJSON: { "time": "2016-09-25T18:57:58", "match": true, "host": "192.0.2.1", "desc": "Systemd dual time with prefix - 1st expr" }
+2016-09-25T18:57:58.150982 www.srv.tld freeswitch[122921]: 2016-09-25 18:57:58.150982 [WARNING] sofia_reg.c:2889 Can't find user [201@::1] from 192.0.2.1
+# failJSON: { "time": "2016-09-25T18:57:58", "match": true, "host": "192.0.2.2", "desc": "Systemd dual time with prefix - 2nd expr" }
+2016-09-25T18:57:58.150982 www.srv.tld freeswitch[122921]: 2016-09-25 18:57:58.150982 [WARNING] sofia_reg.c:1720 SIP auth failure (INVITE) on sofia profile 'sipinterface_1' for [9810972597751739@::1] from ip 192.0.2.2
\ No newline at end of file

From 84fe55b99b8010c1951fea021a886a6cbbc4de7d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 28 Sep 2016 21:17:42 +0200
Subject: [PATCH 222/623] [temp commit] 2nd try to optimize
 datedetector/datetemplate functionality (almost ready, needs fine tuning)

---
 fail2ban/client/fail2banregex.py              |  40 +++--
 fail2ban/server/datedetector.py               | 105 ++++++-------
 fail2ban/server/datetemplate.py               |  45 +++---
 fail2ban/server/mytime.py                     |  20 ++-
 fail2ban/server/strptime.py                   |  51 +++++-
 fail2ban/tests/datedetectortestcase.py        | 147 +++++++++++++++++-
 fail2ban/tests/fail2banregextestcase.py       |   2 +-
 fail2ban/tests/files/logs/zzz-generic-example |   8 +-
 fail2ban/tests/filtertestcase.py              |   6 +-
 fail2ban/tests/misctestcase.py                |  89 -----------
 fail2ban/tests/servertestcase.py              |   2 +-
 fail2ban/tests/utils.py                       |  10 +-
 12 files changed, 305 insertions(+), 220 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 4096ac4e..6680b097 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -122,15 +122,15 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 	p.add_options([
 		Option("-d", "--datepattern",
 			   help="set custom pattern used to match date/times"),
-		Option("-e", "--encoding",
+		Option("-e", "--encoding", default=PREFER_ENC,
 			   help="File encoding. Default: system locale"),
-		Option("-r", "--raw", action='store_true',
+		Option("-r", "--raw", action='store_true', default=False,
 			   help="Raw hosts, don't resolve dns"),
 		Option("--usedns", action='store', default=None,
 			   help="DNS specified replacement of tags <HOST> in regexp "
 			        "('yes' - matches all form of hosts, 'no' - IP addresses only)"),
 		Option("-L", "--maxlines", type=int, default=0,
-			   help="maxlines for multi-line regex"),
+			   help="maxlines for multi-line regex."),
 		Option("-m", "--journalmatch",
 			   help="journalctl style matches overriding filter file. "
 			   "\"systemd-journal\" only"),
@@ -143,6 +143,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   help="Increase verbosity"),
 		Option("--verbosity", action="store", dest="verbose", type=int,
 			   help="Set numerical level of verbosity (0..4)"),
+		Option("--verbose-date", "--VD", action='store_true',
+			   help="Verbose date patterns/regex in output"),
 		Option("-D", "--debuggex", action='store_true',
 			   help="Produce debuggex.com urls for debugging there"),
 		Option("--print-no-missed", action='store_true',
@@ -215,14 +217,8 @@ class LineStats(object):
 class Fail2banRegex(object):
 
 	def __init__(self, opts):
-		self._verbose = opts.verbose
-		self._debuggex = opts.debuggex
-		self._maxlines = 20
-		self._print_no_missed = opts.print_no_missed
-		self._print_no_ignored = opts.print_no_ignored
-		self._print_all_matched = opts.print_all_matched
-		self._print_all_missed = opts.print_all_missed
-		self._print_all_ignored = opts.print_all_ignored
+		# set local protected memebers from given options:
+		self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems()))
 		self._maxlines_set = False		  # so we allow to override maxlines in cmdline
 		self._datepattern_set = False
 		self._journalmatch = None
@@ -236,23 +232,20 @@ class Fail2banRegex(object):
 
 		if opts.maxlines:
 			self.setMaxLines(opts.maxlines)
+		else:
+			self._maxlines = 20
 		if opts.journalmatch is not None:
 			self.setJournalMatch(opts.journalmatch.split())
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
-		if opts.encoding:
-			self.encoding = opts.encoding
-		else:
-			self.encoding = PREFER_ENC
-		self.raw = True if opts.raw else False
 		if opts.usedns:
 			self._filter.setUseDns(opts.usedns)
 
 	def decode_line(self, line):
-		return FileContainer.decode_line('<LOG>', self.encoding, line)
+		return FileContainer.decode_line('<LOG>', self._encoding, line)
 
 	def encode_line(self, line):
-		return line.encode(self.encoding, 'ignore')
+		return line.encode(self._encoding, 'ignore')
 
 	def setDatePattern(self, pattern):
 		if not self._datepattern_set:
@@ -350,7 +343,7 @@ class Fail2banRegex(object):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
-			line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw)
+			line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self._raw)
 			for match in ret:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
@@ -479,8 +472,11 @@ class Fail2banRegex(object):
 			out = []
 			for template in self._filter.dateDetector.templates:
 				if self._verbose or template.hits:
-					out.append("[%d] %s" % (
-						template.hits, template.name))
+					out.append("[%d] %s" % (template.hits, template.name))
+					if self._verbose_date:
+						out.append("    # weight: %3s, pattern: %s" % (
+							template.weight, getattr(template, 'pattern', ''),))
+						out.append("    # regex:   %s" % (getattr(template, 'regex', ''),))
 			pprint_list(out, "[# of hits] date format")
 
 		output( "\nLines: %s" % self._line_stats, )
@@ -518,7 +514,7 @@ class Fail2banRegex(object):
 			try:
 				hdlr = open(cmd_log, 'rb')
 				output( "Use         log file : %s" % cmd_log )
-				output( "Use         encoding : %s" % self.encoding )
+				output( "Use         encoding : %s" % self._encoding )
 				test_lines = self.file_lines_gen(hdlr)
 			except IOError as e:
 				output( e )
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index b1f97af7..d26743a3 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -60,16 +60,15 @@ class DateDetectorCache(object):
 			# exact given template with word benin-end boundary:
 			template = DatePatternRegex(template)
 		# additional template, that prefers datetime at start of a line (safety+performance feature):
-		template2 = copy.copy(template)
-		if hasattr(template, 'pattern'):
-			regex = template.pattern
-			wordEnd = True
-		else:
-			regex = template.regex
-			wordEnd = False
-		template2.setRegex(regex, wordBegin='start', wordEnd=wordEnd)
-		if template2.name != template.name:
-			self.__templates.append(template2)
+		if 0 and hasattr(template, 'regex'):
+			template2 = copy.copy(template)
+			regex = getattr(template, 'pattern', template.regex)
+			template2.setRegex(regex, wordBegin='start', wordEnd=True)
+			if template2.name != template.name:
+				# increase weight of such templates, because they should be always
+				# preferred in template sorting process (bubble up):
+				template2.weight = 100
+				self.__templates.append(template2)
 		# add template:
 		self.__templates.append(template)
 
@@ -80,35 +79,35 @@ class DateDetectorCache(object):
 		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59
 		# simple date: 2005/01/23 21:59:59 
 		# custom for syslog-ng 2006.12.21 06:43:20
-		self._cacheTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
 		# 20050123T215959, 20050123 215959
-		self._cacheTemplate("%Y%Em%Ed[T ]%EH%EM%ES(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
 		# prefixed with optional time zone (monit):
 		# PDT Apr 16 21:05:29
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
-		self._cacheTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?")
+		self._cacheTemplate("(?:%a )?%b %d %ExY %H:%M:%S(?:\.%f)?")
 		# simple date too (from x11vnc): 23/01/2005 21:59:59 
 		# and with optional year given by 2 digits: 23/01/05 21:59:59 
 		# (See http://bugs.debian.org/537610)
 		# 17-07-2008 17:23:25
-		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S")
+		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
 		# Apache format optional time zone:
 		# [31/Oct/2006:09:22:55 -0000]
 		# 26-Jul-2007 15:20:52
 		# named 26-Jul-2007 15:20:52.252
 		# roundcube 26-Jul-2007 15:20:52 +0200
-		self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
+		self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
 		# CPanel 05/20/2008:01:57:39
-		self._cacheTemplate("%m/%d/%Y:%H:%M:%S")
+		self._cacheTemplate("%m/%d/%ExY:%H:%M:%S")
 		# 01-27-2012 16:22:44.252
 		# subseconds explicit to avoid possible %m<->%d confusion
-		# with previous ("%d-%m-%Y %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S")
-		self._cacheTemplate("%m-%d-%Y %H:%M:%S(?:\.%f)?")
+		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
+		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
 		# TAI64N
 		self._cacheTemplate(DateTai64n())
 		# Epoch
@@ -116,13 +115,13 @@ class DateDetectorCache(object):
 		# Only time information in the log
 		self._cacheTemplate("^%H:%M:%S")
 		# <09/16/08@05:03:30>
-		self._cacheTemplate("^<%m/%d/%y@%H:%M:%S>")
+		self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>")
 		# MySQL: 130322 11:46:11
-		self._cacheTemplate("%y%Em%Ed  ?%H:%M:%S")
+		self._cacheTemplate("%Exy%Exm%Exd  ?%H:%M:%S")
 		# Apache Tomcat
-		self._cacheTemplate("%b %d, %Y %I:%M:%S %p")
+		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("^%b-%d-%y %H:%M:%S")
+		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
 
 
 class DateDetectorTemplate(object):
@@ -218,14 +217,14 @@ class DateDetector(object):
 		"""
 		i = 0
 		with self.__lock:
-			for ddtemplate in self.__templates:
-				template = ddtemplate.template
+			for ddtempl in self.__templates:
+				template = ddtempl.template
 				match = template.matchDate(line)
-				if not match is None:
+				if match is not None:
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "Matched time template %s", template.name)
-					ddtemplate.hits += 1
-					ddtemplate.lastUsed = time.time()
+					ddtempl.hits += 1
+					ddtempl.lastUsed = time.time()
 					# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
 					if i:
 						self._reorderTemplate(i)
@@ -254,32 +253,21 @@ class DateDetector(object):
 			The Unix timestamp returned from the first successfully matched
 			template or None if not found.
 		"""
-		if timeMatch:
-			template = timeMatch[1]
-			if template is not None:
-				try:
-					date = template.getDate(line, timeMatch[0])
-					if date is not None:
-						if logSys.getEffectiveLevel() <= logLevel:
-							logSys.log(logLevel, "Got time %f for %r using template %s",
-								date[0], date[1].group(), template.name)
-						return date
-				except ValueError:
-					return None
-		with self.__lock:
-			for ddtemplate in self.__templates:
-				template = ddtemplate.template
-				try:
-					date = template.getDate(line)
-					if date is None:
-						continue
+		# search match for all specified templates:
+		if timeMatch is None:
+			timeMatch = self.matchTime(line)
+		# convert:
+		template = timeMatch[1]
+		if template is not None:
+			try:
+				date = template.getDate(line, timeMatch[0])
+				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "Got time %f for %r using template %s", 
+						logSys.log(logLevel, "Got time %f for %r using template %s",
 							date[0], date[1].group(), template.name)
 					return date
-				except ValueError: # pragma: no cover
-					pass
-			return None
+			except ValueError:
+				return None
 
 	def _reorderTemplate(self, num):
 		"""Reorder template (bubble up) in template list if hits grows enough.
@@ -291,16 +279,16 @@ class DateDetector(object):
 		"""
 		if num:
 			templates = self.__templates
-			template = templates[num]
+			ddtempl = templates[num]
 		  ## current hits and time the template was long unused:
-			untime = template.lastUsed - self.__unusedTime
-			hits = template.hits
+			untime = ddtempl.lastUsed - self.__unusedTime
+			hits = ddtempl.hits * ddtempl.template.weight
 			## try to move faster (first 2 if it still unused, or half of part to current template position):
-			phits = 0
 			for pos in (0, 1, num // 2):
 				phits = templates[pos].hits
-				if not phits:
+				if not phits: # if we've found an unused
 					break
+			phits *= templates[pos].template.weight
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
 			if not phits or hits > phits + 5 or templates[pos].lastUsed < untime:
@@ -308,8 +296,9 @@ class DateDetector(object):
 				if hits <= phits and templates[pos].lastUsed > untime:
 					pos = num-1
 					## if still smaller and template at position used, don't move:
-					if hits < templates[pos].hits and templates[pos].lastUsed > untime:
+					phits = templates[pos].hits * templates[pos].template.weight
+					if hits < phits and templates[pos].lastUsed > untime:
 						return
-				templates[pos], templates[num] = template, templates[pos]
+				templates[pos], templates[num] = ddtempl, templates[pos]
 
 
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 5fcc16a4..6548e5ee 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -32,6 +32,9 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
+RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$|\*\*)$')
+RE_DEL_WRD_BOUNDS =   re.compile(r'^\*\*|(?<!\\)\*\*$')
 
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
@@ -46,20 +49,11 @@ class DateTemplate(object):
 	"""
 
 	def __init__(self):
-		self._name = ""
+		self.name = ""
+		self.weight = 1
 		self._regex = ""
 		self._cRegex = None
 
-	@property
-	def name(self):
-		"""Name assigned to template.
-		"""
-		return self._name
-
-	@name.setter
-	def name(self, name):
-		self._name = name
-
 	def getRegex(self):
 		return self._regex
 
@@ -73,10 +67,12 @@ class DateTemplate(object):
 		wordBegin : bool
 			Defines whether the regex should be modified to search at beginning of a
 			word, by adding special boundary r'(?=^|\b|\W)' to start of regex.
+			Can be disabled with specifying of ** at front of regex.
 			Default True.
 		wordEnd : bool
 			Defines whether the regex should be modified to search at end of a word,
 			by adding special boundary r'(?=\b|\W|$)' to end of regex.
+			Can be disabled with specifying of ** at end of regex.
 			Default True.
 
 		Raises
@@ -85,12 +81,16 @@ class DateTemplate(object):
 			If regular expression fails to compile
 		"""
 		regex = regex.strip()
-		if wordBegin and not re.search(r'^\^', regex):
-			regex = (r'(?<=^|\b)' if wordBegin != 'start' else r"^(?<=\W)?") + regex
-			self._name = ('[*WD-BEG]' if wordBegin != 'start' else '[^LN-BEG]') + self._name
-		if wordEnd and not re.search(r'\$$', regex):
+		# if word or line start boundary:
+		if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
+			regex = (r'(?=^|\b|\W)' if wordBegin != 'start' else r"(?:^|(?<=^\W)|(?<=^\W{2}))") + regex
+			self.name = ('{*WD-BEG}' if wordBegin != 'start' else '{^LN-BEG}') + self.name
+		# if word end boundary:
+		if wordEnd and not RE_NO_WRD_BOUND_END.search(regex):
 			regex += r'(?=\b|\W|$)'
-			self._name += ('[*WD-END]' if wordEnd else '')
+			self.name += ('{*WD-END}' if wordEnd else '')
+		# remove possible special pattern "**" in front and end of regex:
+		regex = RE_DEL_WRD_BOUNDS.sub('', regex)
 		self._regex = regex
 
 	regex = property(getRegex, setRegex, doc=
@@ -181,15 +181,8 @@ class DatePatternRegex(DateTemplate):
 	pattern
 	"""
 	
-	_patternRE = re.compile(getTimePatternRE())
-	_patternName = {
-		'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
-		'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
-		'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
-		'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%",
-		'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name"}
-	for _key in set(timeRE) - set(_patternName): # may not have them all...
-		_patternName[_key] = "%%%s" % _key
+	_patternRE, _patternName = getTimePatternRE()
+	_patternRE = re.compile(_patternRE)
 
 	def __init__(self, pattern=None, **kwargs):
 		super(DatePatternRegex, self).__init__()
@@ -216,7 +209,7 @@ class DatePatternRegex(DateTemplate):
 	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
 		self._pattern = pattern
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
-		self._name = fmt % self._patternName
+		self.name = fmt % self._patternName
 		super(DatePatternRegex, self).setRegex(fmt % timeRE, wordBegin, wordEnd)
 
 	def getDate(self, line, dateMatch=None):
diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py
index 816b04c4..49199887 100644
--- a/fail2ban/server/mytime.py
+++ b/fail2ban/server/mytime.py
@@ -41,6 +41,21 @@ class MyTime:
 	"""
 
 	myTime = None
+	alternateNowTime = None
+	alternateNow = None
+
+	@staticmethod
+	def setAlternateNow(t):
+		"""Set current time.
+
+		Use None in order to always get the real current time.
+
+		@param t the time to set or None
+		"""
+
+		MyTime.alternateNowTime = t
+		MyTime.alternateNow = \
+			datetime.datetime.fromtimestamp(t) if t is not None else None
 
 	@staticmethod
 	def setTime(t):
@@ -84,8 +99,9 @@ class MyTime:
 		"""
 		if MyTime.myTime is None:
 			return datetime.datetime.now()
-		else:
-			return datetime.datetime.fromtimestamp(MyTime.myTime)
+		if MyTime.myTime == MyTime.alternateNowTime:
+			return MyTime.alternateNow
+		return datetime.datetime.fromtimestamp(MyTime.myTime)
 
 	@staticmethod
 	def localtime(x=None):
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index c6a1f37e..2012aae6 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -26,24 +26,59 @@ from .mytime import MyTime
 
 locale_time = LocaleTime()
 timeRE = TimeRE()
+
+def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
+	""" Build century regex for last year and the next years (distance).
+		
+	Thereby respect possible run in the test-cases (alternate date used there)
+	"""
+	cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t]
+	exprset = set( cent(now[0].year + i) for i in (-1, distance) )
+	if len(now) and now[1]:
+		exprset |= set( cent(now[1].year + i) for i in (-1, distance) )
+	return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset)
+
 #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
 #timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
 timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
 
-# Extend build-in TimeRE with some exact (two-digit) patterns:
-timeRE['Ed'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
-timeRE['Em'] = r"(?P<m>1[0-2]|0[1-9])"
-timeRE['EH'] = r"(?P<H>2[0-3]|[0-1]\d)"
-timeRE['EM'] = r"(?P<M>[0-5]\d)"
-timeRE['ES'] = r"(?P<S>6[0-1]|[0-5]\d)"
+# Extend build-in TimeRE with some exact patterns
+# exact two-digit patterns:
+timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
+timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])"
+timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)"
+timeRE['ExM'] = r"(?P<M>[0-5]\d)"
+timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
+# more precise year patterns, within same century of last year and
+# the next 3 years (for possible long uptime of fail2ban); thereby
+# respect possible run in the test-cases (alternate date used there):
+timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3)
+timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3)
+# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates:
+timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))"
 
 def getTimePatternRE():
 	keys = timeRE.keys()
-	return (r"%%(%%|%s|[%s])" % (
+	patt = (r"%%(%%|%s|[%s])" % (
 		"|".join([k for k in keys if len(k) > 1]),
 		"".join([k for k in keys if len(k) == 1]),
 	))
-
+	names = {
+		'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
+		'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
+		'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
+		'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%",
+		'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name",
+		'ExLB': '{^LN-BEG}',
+	}
+	for key in set(keys) - set(names): # may not have them all...
+		if key.startswith('Ex'):
+			kn = names.get(key[2:])
+			if kn:
+				names[key] = "Ex" + kn
+				continue
+		names[key] = "%%%s" % key
+	return (patt, names)
 
 def reGroupDictStrptime(found_dict):
 	"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 013e1e85..109a275f 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -30,7 +30,7 @@ import datetime
 
 from ..server.datedetector import DateDetector
 from ..server import datedetector
-from ..server.datetemplate import DateTemplate
+from ..server.datetemplate import DatePatternRegex, DateTemplate
 from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase
 from ..helpers import getLogger
 
@@ -89,6 +89,10 @@ class DateDetectorTest(LogCaptureTestCase):
 		"""
 		dateUnix = 1106513999.0
 
+		# anchored - matching expression (pattern) is anchored
+		# bound - pattern can be tested using word boundary (e.g. False if contains in front some optional part)
+		# sdate - date string used in test log-line
+		# rdate - if specified, the result match, which differs from sdate
 		for anchored, bound, sdate, rdate in (
 			(False, True,  "Jan 23 21:59:59", None),
 			(False, False, "Sun Jan 23 21:59:59 2005", None),
@@ -113,15 +117,15 @@ class DateDetectorTest(LogCaptureTestCase):
 			(False, True,  "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC)
 			(False, True,  "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ
 			(False, True,  "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local
-			(False, True,  "20050123T215959", None),   #Short ISO
-			(False, True,  "20050123 215959", None),   #Short ISO
+			(False, True,  "20050123T215959", None),   #Short ISO with T
+			(False, True,  "20050123 215959", None),   #Short ISO with space
 			(True,  True,  "<01/23/05@21:59:59>", None),
 			(False, True,  "050123 21:59:59", None), # MySQL
 			(True,  True,  "Jan-23-05 21:59:59", None), # ASSP like
 			(False, True,  "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat
 			(True,  True,  "1106513999", None), # Regular epoch
 			(True,  True,  "1106513999.000", None), # Regular epoch with millisec
-			(True,  True,  "[1106513999.000]", "1106513999.000"), # epoch squared
+			(True,  True,  "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match)
 			(False, True,  "audit(1106513999.000:987)", "1106513999.000"), # SELinux
 		):
 			logSys.debug('== test %r', (anchored, bound, sdate))
@@ -195,6 +199,141 @@ class DateDetectorTest(LogCaptureTestCase):
 			self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
 
 
+iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
+
+class CustomDateFormatsTest(unittest.TestCase):
+
+	def testIso8601(self):
+		date = datetime.datetime.utcfromtimestamp(
+			iso8601.getDate("2007-01-25T12:00:00Z")[0])
+		self.assertEqual(
+			date,
+			datetime.datetime(2007, 1, 25, 12, 0))
+		self.assertRaises(TypeError, iso8601.getDate, None)
+		self.assertRaises(TypeError, iso8601.getDate, date)
+
+		self.assertEqual(iso8601.getDate(""), None)
+		self.assertEqual(iso8601.getDate("Z"), None)
+
+		self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None)
+		self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None)
+		date = datetime.datetime.utcfromtimestamp(
+			iso8601.getDate("2007-01-25T12:00:00+0400")[0])
+		self.assertEqual(
+			date,
+			datetime.datetime(2007, 1, 25, 8, 0))
+		date = datetime.datetime.utcfromtimestamp(
+			iso8601.getDate("2007-01-25T12:00:00+04:00")[0])
+		self.assertEqual(
+			date,
+			datetime.datetime(2007, 1, 25, 8, 0))
+		date = datetime.datetime.utcfromtimestamp(
+			iso8601.getDate("2007-01-25T12:00:00-0400")[0])
+		self.assertEqual(
+			date,
+			datetime.datetime(2007, 1, 25, 16, 0))
+		date = datetime.datetime.utcfromtimestamp(
+			iso8601.getDate("2007-01-25T12:00:00-04")[0])
+		self.assertEqual(
+			date,
+			datetime.datetime(2007, 1, 25, 16, 0))
+
+	def testAmbiguousDatePattern(self):
+		defDD = DateDetector()
+		defDD.addDefaultTemplate()
+		for (matched, dp, line) in (
+			# positive case:
+			('Jan 23 21:59:59',   None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'),
+			# ambiguous "unbound" patterns (missed):
+			(False,               None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'),
+			(False,               None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'),
+			# ambiguous "no optional year" patterns (matched):
+			('Aug 8 11:25:50',      None, 'Aug 8 11:25:50 20030f2329b8 Authentication failed from 192.0.2.1'),
+			('Aug 8 11:25:50',      None, '[Aug 8 11:25:50] 20030f2329b8 Authentication failed from 192.0.2.1'),
+			('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 20030f2329b8 Authentication failed from 192.0.2.1'),
+			# direct specified patterns:
+			('20:00:00 01.02.2003',    r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'),
+			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'),
+			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
+			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'),
+			('[20:00:00 01.02.2003]',  r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
+			('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'),
+			('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'),
+			('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'),
+			# direct specified patterns (begin/end, missed):
+			(False,                 r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'),
+			(False,                 r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'),
+			# direct specified unbound patterns (no begin/end boundary):
+			('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '192.0.2.1x20:00:00 01.02.2003'),
+			('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '20:00:00 01.02.2003x192.0.2.1'),
+			# pattern enclosed with stars (in comparison to example above):
+			('*20:00:00 01.02.2003*', r'\**%H:%M:%S %d.%m.%Y\**', 'test*20:00:00 01.02.2003*test'),
+			# direct specified patterns (begin/end, matched):
+			('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'),
+			('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'),
+			# wrong year in 1st date, so failed by convert using not precise year (filter used last known date),
+			# in the 2nd and 3th tests (with precise year) it should find correct the 2nd date:
+			(None,                  r'%Y-%Exm-%Exd %ExH:%ExM:%ExS',   "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
+			('2003-12-30 00:00:00', r'%ExY-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
+			('2003-12-30 00:00:00', None,                             "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
+			# wrong date recognized short month/day (unbounded date pattern without separator between parts),
+			# in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date:
+			('200333 010203',   r'%Y%m%d %H%M%S',             "text:200333 010203 | date:20031230 010203"),
+			('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
+			('20031230 010203', None,                         "text:200333 010203 | date:20031230 010203"),
+			# Explicit bound in start of the line using %ExLB key,
+			# (negative) in the 1st case without line begin boundary - wrong date may be found,
+			# (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found)
+			# (positive) and in 3th case with line begin boundary - find the correct date
+			("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS",      "00001230 010203 - 20030101 000000"),
+			(None,              "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
+			("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
+			# Explicit bound in start of the line using %ExLB key, 
+			# up to 2 non-alphanumeric chars front, ** - no word boundary on the right
+			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
+			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
+			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
+			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
+		):
+			logSys.debug('== test: %r', (matched, dp, line))
+			if dp is None:
+				dd = defDD
+			else:
+				dp = DatePatternRegex(dp)
+				dd = DateDetector()
+				dd.appendTemplate(dp)
+			date = dd.getTime(line)
+			if matched:
+				self.assertTrue(date)
+				self.assertEqual(matched, date[1].group())
+			else:
+				self.assertEqual(date, None)
+
+	# def testAmbiguousUsingOrderedTemplates(self):
+	# 	defDD = DateDetector()
+	# 	defDD.addDefaultTemplate()
+	# 	for (matched, dp, line) in (
+	# 		# wrong date recognized short month/day (unbounded date pattern without separator),
+	# 		# in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date:
+	# 		('200333 010203',   r'%Y%m%d %H%M%S',             "text:200333 010203 | date:20031230 010203"),
+	# 		('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
+	# 		('20031230 010203', None,                         "text:200333 010203 | date:20031230 010203"),
+	# 	):
+	# 		logSys.debug('== test: %r', (matched, dp, line))
+	# 		if dp is None:
+	# 			dd = defDD
+	# 		else:
+	# 			dp = DatePatternRegex(dp)
+	# 			dd = DateDetector()
+	# 			dd.appendTemplate(dp)
+	# 		date = dd.getTime(line)
+	# 		if matched:
+	# 			self.assertTrue(date)
+	# 			self.assertEqual(matched, date[1].group())
+	# 		else:
+	# 			self.assertEqual(date, None)
+
+
 #	def testDefaultTempate(self):
 #		self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 #		self.__datedetector.setDefaultPattern("%b %d %H:%M:%S")
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 4445fe6b..c62377a9 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -178,7 +178,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testVerbose(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
-			"--verbose", "--print-no-missed",
+			"--verbose", "--verbose-date", "--print-no-missed",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
 		)
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index 2044c387..51d3974c 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -30,8 +30,8 @@ Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-
 # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
 [Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
 
-# -- wrong time direct in journal-line (used last known date):
-# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" }
+# -- wrong time direct in journal-line (using precise year pattern):
+# failJSON: { "match": false}
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.1
 # -- wrong time after newline in message (plist without escaped newlines):
 # failJSON: { "match": false }
@@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
              applicationDate = "0000-12-30 00:00:00 +0000";
 # failJSON: { "match": false }
 }
-# -- wrong time direct in journal-line (used last known date):
-# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
+# -- wrong time direct in journal-line (using precise year pattern):
+# failJSON: { "match": false}
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2
 
 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" }
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 2b57ce47..6c1a637c 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -283,10 +283,10 @@ class BasicFilter(unittest.TestCase):
 	def testGetSetDatePattern(self):
 		self.assertEqual(self.filter.getDatePattern(),
 			(None, "Default Detectors"))
-		self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z")
+		self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z **")
 		self.assertEqual(self.filter.getDatePattern(),
-			("^%Y-%m-%d-%H%M%S.%f %z",
-			"^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset"))
+			("^%Y-%m-%d-%H%M%S.%f %z **",
+			"^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **"))
 
 	def testAssertWrongTime(self):
 		self.assertRaises(AssertionError, 
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 450904d5..908e4f6c 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -23,13 +23,11 @@ __license__ = "GPL"
 
 import logging
 import os
-import re
 import sys
 import unittest
 import tempfile
 import shutil
 import fnmatch
-import datetime
 from glob import glob
 from StringIO import StringIO
 
@@ -37,8 +35,6 @@ from utils import LogCaptureTestCase, logSys as DefLogSys
 
 from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, uni_decode
 from ..helpers import splitwords
-from ..server.datedetector import DateDetector
-from ..server.datetemplate import DatePatternRegex
 from ..server.mytime import MyTime
 
 
@@ -320,91 +316,6 @@ class TestsUtilsTest(LogCaptureTestCase):
 			self.assertRaisesRegexp(Exception, 'not all arguments converted', lambda: logSys.debug('test', 1, 2, 3))
 
 
-iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
-
-
-class CustomDateFormatsTest(unittest.TestCase):
-
-	def testIso8601(self):
-		date = datetime.datetime.utcfromtimestamp(
-			iso8601.getDate("2007-01-25T12:00:00Z")[0])
-		self.assertEqual(
-			date,
-			datetime.datetime(2007, 1, 25, 12, 0))
-		self.assertRaises(TypeError, iso8601.getDate, None)
-		self.assertRaises(TypeError, iso8601.getDate, date)
-
-		self.assertEqual(iso8601.getDate(""), None)
-		self.assertEqual(iso8601.getDate("Z"), None)
-
-		self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None)
-		self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None)
-		date = datetime.datetime.utcfromtimestamp(
-			iso8601.getDate("2007-01-25T12:00:00+0400")[0])
-		self.assertEqual(
-			date,
-			datetime.datetime(2007, 1, 25, 8, 0))
-		date = datetime.datetime.utcfromtimestamp(
-			iso8601.getDate("2007-01-25T12:00:00+04:00")[0])
-		self.assertEqual(
-			date,
-			datetime.datetime(2007, 1, 25, 8, 0))
-		date = datetime.datetime.utcfromtimestamp(
-			iso8601.getDate("2007-01-25T12:00:00-0400")[0])
-		self.assertEqual(
-			date,
-			datetime.datetime(2007, 1, 25, 16, 0))
-		date = datetime.datetime.utcfromtimestamp(
-			iso8601.getDate("2007-01-25T12:00:00-04")[0])
-		self.assertEqual(
-			date,
-			datetime.datetime(2007, 1, 25, 16, 0))
-
-	def testAmbiguousDatePattern(self):
-		defDD = DateDetector()
-		defDD.addDefaultTemplate()
-		logSys = DefLogSys
-		for (matched, dp, line) in (
-			# positive case:
-			('Jan 23 21:59:59',   None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'),
-			# ambiguous "unbound" patterns (missed):
-			(False,               None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'),
-			(False,               None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'),
-			# ambiguous "no optional year" patterns (matched):
-			('Aug 8 11:25:50',      None, 'Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1'),
-			('Aug 8 11:25:50',      None, '[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1'),
-			('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 14430f2329b8 Authentication failed from 192.0.2.1'),
-			# direct specified patterns:
-			('20:00:00 01.02.2003',    r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'),
-			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'),
-			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
-			('[20:00:00 01.02.2003]',  r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'),
-			('[20:00:00 01.02.2003]',  r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
-			('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'),
-			('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'),
-			('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'),
-			# direct specified patterns (begin/end, missed):
-			(False,                 r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'),
-			(False,                 r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'),
-			# direct specified patterns (begin/end, matched):
-			('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'),
-			('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'),
-		):
-			logSys.debug('== test: %r', (matched, dp, line))
-			if dp is None:
-				dd = defDD
-			else:
-				dp = DatePatternRegex(dp)
-				dd = DateDetector()
-				dd.appendTemplate(dp)
-			date = dd.getTime(line)
-			if matched:
-				self.assertTrue(date)
-				self.assertEqual(matched, date[1].group())
-			else:
-				self.assertEqual(date, None)
-
-
 class MyTimeTest(unittest.TestCase):
 
 	def testStr2Seconds(self):
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 56c85e94..d4320257 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -298,7 +298,7 @@ class Transmitter(TransmitterBase):
 
 	def testDatePattern(self):
 		self.setGetTest("datepattern", "%%%Y%m%d%H%M%S",
-			("%%%Y%m%d%H%M%S", "%YearMonthDay24hourMinuteSecond"),
+			("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"),
 			jail=self.jailName)
 		self.setGetTest(
 			"datepattern", "Epoch", (None, "Epoch"), jail=self.jailName)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index f60dfd1f..22079456 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -48,6 +48,8 @@ from ..version import version
 
 logSys = getLogger(__name__)
 
+TEST_NOW = 1124013600
+
 CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None)
 
 if not CONFIG_DIR:
@@ -257,6 +259,10 @@ def initTests(opts):
 		def F2B_SkipIfNoNetwork():
 			raise unittest.SkipTest('Skip test because of "--no-network"')
 		unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork
+
+	# set alternate now for time related test cases:
+	MyTime.setAlternateNow(TEST_NOW)
+
 	# precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737):
 	c = DNSUtils.CACHE_ipToName
 	for i in xrange(255):
@@ -289,7 +295,7 @@ def setUpMyTime():
 	# yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match
 	os.environ['TZ'] = 'Europe/Zurich'
 	time.tzset()
-	MyTime.setTime(1124013600)
+	MyTime.setTime(TEST_NOW)
 
 
 def tearDownMyTime():
@@ -384,7 +390,6 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(misctestcase.HelpersTest))
 	tests.addTest(unittest.makeSuite(misctestcase.SetupTest))
 	tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest))
-	tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest))
 	tests.addTest(unittest.makeSuite(misctestcase.MyTimeTest))
 	# Database
 	tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest))
@@ -404,6 +409,7 @@ def gatherTests(regexps=None, opts=None):
 
 	# DateDetector
 	tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest))
+	tests.addTest(unittest.makeSuite(datedetectortestcase.CustomDateFormatsTest))
 	# Filter Regex tests with sample logs
 	tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex))
 

From 75a5440acf96c1b3fda37eef9b86de92ee3b1c17 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 30 Sep 2016 20:37:02 +0200
Subject: [PATCH 223/623] extends date detector template with distance
 (position of match in log-line), to prevent grave collision using (re)ordered
 template list (e.g. find-spot of wrong date-match inside foreign input,
 misleading date patterns by ambiguous formats, etc.); By change of the
 distance (e.g. another format found), the pattern with smallest distance will
 be always preferred now. To speedup (template lookup) resp. minimize of list
 reorder counts, the distance will be used as divider factor of the template
 weight by the templates comparison.

---
 fail2ban/server/datedetector.py        | 81 ++++++++++++++++++--------
 fail2ban/tests/datedetectortestcase.py | 47 ++++++++-------
 2 files changed, 81 insertions(+), 47 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index d26743a3..63ae3692 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -129,11 +129,13 @@ class DateDetectorTemplate(object):
 
 	Prevents collectively usage of hits/lastUsed in cached templates
 	"""
-	__slots__ = ('template', 'hits', 'lastUsed')
+	__slots__ = ('template', 'hits', 'lastUsed', 'distance')
 	def __init__(self, template):
 		self.template = template
 		self.hits = 0
 		self.lastUsed = 0
+		# the last distance to date-match within the log file:
+		self.distance = 0x7fffffff
 
 	def __getattr__(self, name):
 		""" Returns attribute of template (called for parameters not in slots)
@@ -156,6 +158,8 @@ class DateDetector(object):
 		self.__known_names = set()
 		# time the template was long unused (currently 300 == 5m):
 		self.__unusedTime = 300
+		# first free place:
+		self.__firstUnused = 0
 
 	def _appendTemplate(self, template):
 		name = template.name
@@ -215,22 +219,52 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
+		match = None
 		i = 0
+		found = None, 0x7fffffff, -1
 		with self.__lock:
 			for ddtempl in self.__templates:
 				template = ddtempl.template
 				match = template.matchDate(line)
 				if match is not None:
+					distance = max(1, match.start() + 1)
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "Matched time template %s", template.name)
-					ddtempl.hits += 1
-					ddtempl.lastUsed = time.time()
-					# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
-					if i:
-						self._reorderTemplate(i)
-					# return tuple with match and template reference used for parsing:
-					return (match, template)
+						logSys.log(logLevel, "  matched time template #%r (at %r <= %r) %s",
+							i, distance, ddtempl.distance, template.name)
+					## [grave] if distance changed, possible date-match was found somewhere 
+					## in body of message, so save this template, and search further:
+					if distance > ddtempl.distance and len(self.__templates) > 1:
+						if logSys.getEffectiveLevel() <= logLevel:
+							logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
+						## shortest of both:
+						if distance < found[1]:
+							found = match, distance, i
+						## search further:
+						match = None
+						i += 1
+						continue
+					## winner - stop search:
+					break
 				i += 1
+			# check other template was found (use this one with shortest distance):
+			if match is None and found[0]:
+				match, distance, i = found
+				ddtempl = self.__templates[i]
+				template = ddtempl.template
+			# we've winner, incr hits, set distance, usage, reorder, etc:
+			if match is not None:
+				ddtempl.hits += 1
+				ddtempl.distance = distance
+				ddtempl.lastUsed = time.time()
+				if self.__firstUnused == i:
+					self.__firstUnused += 1
+				# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
+				if i:
+					logSys.log(logLevel, "  -> reorder template #%r, hits: %r", i, ddtempl.hits)
+					self._reorderTemplate(i)
+				# return tuple with match and template reference used for parsing:
+				return (match, template)
+
 		# not found:
 		return (None, None)
 
@@ -263,7 +297,7 @@ class DateDetector(object):
 				date = template.getDate(line, timeMatch[0])
 				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "Got time %f for %r using template %s",
+						logSys.log(logLevel, "  got time %f for %r using template %s",
 							date[0], date[1].group(), template.name)
 					return date
 			except ValueError:
@@ -282,23 +316,24 @@ class DateDetector(object):
 			ddtempl = templates[num]
 		  ## current hits and time the template was long unused:
 			untime = ddtempl.lastUsed - self.__unusedTime
-			hits = ddtempl.hits * ddtempl.template.weight
-			## try to move faster (first 2 if it still unused, or half of part to current template position):
-			for pos in (0, 1, num // 2):
-				phits = templates[pos].hits
-				if not phits: # if we've found an unused
-					break
-			phits *= templates[pos].template.weight
+			weight = ddtempl.hits * ddtempl.template.weight / ddtempl.distance
+			## try to move faster (first if unused available, or half of part to current template position):
+			pos = self.__firstUnused if self.__firstUnused < num else num // 2
+			pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
-			if not phits or hits > phits + 5 or templates[pos].lastUsed < untime:
+			logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight)
+			if not pweight or weight > pweight + 5 or templates[pos].lastUsed < untime:
 				## if not larger (and target position recently used) - move slow (exact 1 position):
-				if hits <= phits and templates[pos].lastUsed > untime:
+				if weight <= pweight and templates[pos].lastUsed > untime:
 					pos = num-1
 					## if still smaller and template at position used, don't move:
-					phits = templates[pos].hits * templates[pos].template.weight
-					if hits < phits and templates[pos].lastUsed > untime:
+					pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
+					logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight)
+					if weight < pweight and templates[pos].lastUsed > untime:
 						return
 				templates[pos], templates[num] = ddtempl, templates[pos]
-
-
+				logSys.log(logLevel, "  -> moved template #%r -> #%r", num, pos)
+				## correct first unused:
+				if pos == self.__firstUnused:
+					self.__firstUnused += 1
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 109a275f..9616edbb 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -198,6 +198,29 @@ class DateDetectorTest(LogCaptureTestCase):
 			self.assertRaises(Exception, t.getDate, '')
 			self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
 
+	def testAmbiguousInOrderedTemplates(self):
+		dd = DateDetector()
+		dd.addDefaultTemplate()
+		for (debit, line, cnt) in (
+			("2003-03-07 17:05:01",        "some free text 2003-03-07 17:05:01 test ...", 15),
+			# distance collision detection (date from foreign input should not be found):
+			("030324  0:04:00",            "server mysqld[1000]: 030324  0:04:00 [Warning] Access denied ..."
+																				" foreign-input just some free text 2003-03-07 17:05:01 test", 10),
+			# distance collision detection (first date should be found):
+			("Sep 16 21:30:26",            "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 10),
+			# just to test sorting:
+			("2005-10-07 06:09:42",        "server mysqld[5906]: 2005-10-07 06:09:42 5907 [Warning] Access denied", 20),
+			("2005-10-08T15:26:18.237955", "server mysqld[5906]: 2005-10-08T15:26:18.237955 6 [Note] Access denied", 20),
+			# date format changed again:
+			("051009 10:05:30",            "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 20),
+		):
+			logSys.debug('== test: %r', (debit, line, cnt))
+			for i in range(cnt):
+				logSys.debug('Line: %s', line)
+				match, template = dd.matchTime(line)
+				self.assertTrue(match)
+				self.assertEqual(match.group(), debit)
+
 
 iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
 
@@ -309,30 +332,6 @@ class CustomDateFormatsTest(unittest.TestCase):
 			else:
 				self.assertEqual(date, None)
 
-	# def testAmbiguousUsingOrderedTemplates(self):
-	# 	defDD = DateDetector()
-	# 	defDD.addDefaultTemplate()
-	# 	for (matched, dp, line) in (
-	# 		# wrong date recognized short month/day (unbounded date pattern without separator),
-	# 		# in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date:
-	# 		('200333 010203',   r'%Y%m%d %H%M%S',             "text:200333 010203 | date:20031230 010203"),
-	# 		('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
-	# 		('20031230 010203', None,                         "text:200333 010203 | date:20031230 010203"),
-	# 	):
-	# 		logSys.debug('== test: %r', (matched, dp, line))
-	# 		if dp is None:
-	# 			dd = defDD
-	# 		else:
-	# 			dp = DatePatternRegex(dp)
-	# 			dd = DateDetector()
-	# 			dd.appendTemplate(dp)
-	# 		date = dd.getTime(line)
-	# 		if matched:
-	# 			self.assertTrue(date)
-	# 			self.assertEqual(matched, date[1].group())
-	# 		else:
-	# 			self.assertEqual(date, None)
-
 
 #	def testDefaultTempate(self):
 #		self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")

From b9033d004ed7e4179e82b44d582dafd95b443b67 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 4 Oct 2016 17:25:49 +0200
Subject: [PATCH 224/623] amend distance collision check - always find template
 with shortest distance, also first time (test-case extended); datedetector
 property template without lock, initially placed start-anchored templates at
 begin of template list, small optimization of strptime

---
 fail2ban/server/datedetector.py        | 39 +++++++----
 fail2ban/server/datetemplate.py        |  8 +--
 fail2ban/server/strptime.py            | 92 ++++++++++++++------------
 fail2ban/tests/datedetectortestcase.py |  3 +
 4 files changed, 79 insertions(+), 63 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 63ae3692..e2c14d05 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -46,6 +46,8 @@ class DateDetectorCache(object):
 	def templates(self):
 		"""List of template instances managed by the detector.
 		"""
+		if self.__templates:
+			return self.__templates
 		with self.__lock:
 			if self.__templates:
 				return self.__templates
@@ -60,21 +62,22 @@ class DateDetectorCache(object):
 			# exact given template with word benin-end boundary:
 			template = DatePatternRegex(template)
 		# additional template, that prefers datetime at start of a line (safety+performance feature):
-		if 0 and hasattr(template, 'regex'):
+		if hasattr(template, 'regex'):
 			template2 = copy.copy(template)
 			regex = getattr(template, 'pattern', template.regex)
 			template2.setRegex(regex, wordBegin='start', wordEnd=True)
 			if template2.name != template.name:
 				# increase weight of such templates, because they should be always
 				# preferred in template sorting process (bubble up):
-				template2.weight = 100
-				self.__templates.append(template2)
+				template2.weight = 100.0
+				self.__tmpcache[0].append(template2)
 		# add template:
-		self.__templates.append(template)
+		self.__tmpcache[1].append(template)
 
 	def _addDefaultTemplate(self):
 		"""Add resp. cache Fail2Ban's default set of date templates.
 		"""
+		self.__tmpcache = [], []
 		# ISO 8601, simple date, optional subsecond and timezone:
 		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59
 		# simple date: 2005/01/23 21:59:59 
@@ -122,6 +125,8 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
 		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
+		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
+		del self.__tmpcache
 
 
 class DateDetectorTemplate(object):
@@ -158,6 +163,8 @@ class DateDetector(object):
 		self.__known_names = set()
 		# time the template was long unused (currently 300 == 5m):
 		self.__unusedTime = 300
+		# last known distance:
+		self.__lastDistance = 0
 		# first free place:
 		self.__firstUnused = 0
 
@@ -229,11 +236,14 @@ class DateDetector(object):
 				if match is not None:
 					distance = max(1, match.start() + 1)
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "  matched time template #%r (at %r <= %r) %s",
-							i, distance, ddtempl.distance, template.name)
+						logSys.log(logLevel, "  matched time template #%r (at %r <= %r, %r) %s",
+							i, distance, ddtempl.distance, self.__lastDistance, template.name)
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
-					if distance > ddtempl.distance and len(self.__templates) > 1:
+					if (
+						(distance > ddtempl.distance or distance > self.__lastDistance) and
+						len(self.__templates) > 1
+					):
 						if logSys.getEffectiveLevel() <= logLevel:
 							logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
@@ -254,7 +264,7 @@ class DateDetector(object):
 			# we've winner, incr hits, set distance, usage, reorder, etc:
 			if match is not None:
 				ddtempl.hits += 1
-				ddtempl.distance = distance
+				self.__lastDistance = ddtempl.distance = distance
 				ddtempl.lastUsed = time.time()
 				if self.__firstUnused == i:
 					self.__firstUnused += 1
@@ -322,18 +332,21 @@ class DateDetector(object):
 			pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
-			logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight)
-			if not pweight or weight > pweight + 5 or templates[pos].lastUsed < untime:
+			logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r, hits %r > %r",
+				num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
+			if not pweight or weight > pweight or templates[pos].lastUsed < untime:
 				## if not larger (and target position recently used) - move slow (exact 1 position):
 				if weight <= pweight and templates[pos].lastUsed > untime:
 					pos = num-1
 					## if still smaller and template at position used, don't move:
 					pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
-					logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r", num, pos, weight, pweight)
+					logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r, hits %r > %r",
+						num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
 					if weight < pweight and templates[pos].lastUsed > untime:
 						return
-				templates[pos], templates[num] = ddtempl, templates[pos]
-				logSys.log(logLevel, "  -> moved template #%r -> #%r", num, pos)
+				del templates[num]
+				templates[pos:0] = [ddtempl]
 				## correct first unused:
 				if pos == self.__firstUnused:
 					self.__firstUnused += 1
+				logSys.log(logLevel, "  -> moved template #%r -> #%r", num, pos)
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 6548e5ee..3d116b1e 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -50,7 +50,7 @@ class DateTemplate(object):
 
 	def __init__(self):
 		self.name = ""
-		self.weight = 1
+		self.weight = 1.0
 		self._regex = ""
 		self._cRegex = None
 
@@ -232,11 +232,7 @@ class DatePatternRegex(DateTemplate):
 		if not dateMatch:
 			dateMatch = self.matchDate(line)
 		if dateMatch:
-			groupdict = dict(
-				(key, value)
-				for key, value in dateMatch.groupdict().iteritems()
-				if value is not None)
-			return reGroupDictStrptime(groupdict), dateMatch
+			return reGroupDictStrptime(dateMatch.groupdict()), dateMatch
 
 
 class DateTai64n(DateTemplate):
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 2012aae6..85045ed8 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -80,7 +80,7 @@ def getTimePatternRE():
 		names[key] = "%%%s" % key
 	return (patt, names)
 
-def reGroupDictStrptime(found_dict):
+def reGroupDictStrptime(found_dict, msec=False):
 	"""Return time from dictionary of strptime fields
 
 	This is tweaked from python built-in _strptime.
@@ -109,14 +109,15 @@ def reGroupDictStrptime(found_dict):
 	# weekday and julian defaulted to -1 so as to signal need to calculate
 	# values
 	weekday = julian = -1
-	for group_key in found_dict.keys():
+	for key, val in found_dict.iteritems():
+		if val is None: continue
 		# Directives not explicitly handled below:
 		#   c, x, X
 		#	  handled by making out of other directives
 		#   U, W
 		#	  worthless without day of the week
-		if group_key == 'y':
-			year = int(found_dict['y'])
+		if key == 'y':
+			year = int(val)
 			# Open Group specification for strptime() states that a %y
 			#value in the range of [00, 68] is in the century 2000, while
 			#[69,99] is in the century 1900
@@ -124,20 +125,20 @@ def reGroupDictStrptime(found_dict):
 				year += 2000
 			else:
 				year += 1900
-		elif group_key == 'Y':
-			year = int(found_dict['Y'])
-		elif group_key == 'm':
-			month = int(found_dict['m'])
-		elif group_key == 'B':
-			month = locale_time.f_month.index(found_dict['B'].lower())
-		elif group_key == 'b':
-			month = locale_time.a_month.index(found_dict['b'].lower())
-		elif group_key == 'd':
-			day = int(found_dict['d'])
-		elif group_key == 'H':
-			hour = int(found_dict['H'])
-		elif group_key == 'I':
-			hour = int(found_dict['I'])
+		elif key == 'Y':
+			year = int(val)
+		elif key == 'm':
+			month = int(val)
+		elif key == 'B':
+			month = locale_time.f_month.index(val.lower())
+		elif key == 'b':
+			month = locale_time.a_month.index(val.lower())
+		elif key == 'd':
+			day = int(val)
+		elif key == 'H':
+			hour = int(val)
+		elif key == 'I':
+			hour = int(val)
 			ampm = found_dict.get('p', '').lower()
 			# If there was no AM/PM indicator, we'll treat this like AM
 			if ampm in ('', locale_time.am_pm[0]):
@@ -152,37 +153,38 @@ def reGroupDictStrptime(found_dict):
 				# 12 noon == 12 PM == hour 12
 				if hour != 12:
 					hour += 12
-		elif group_key == 'M':
-			minute = int(found_dict['M'])
-		elif group_key == 'S':
-			second = int(found_dict['S'])
-		elif group_key == 'f':
-			s = found_dict['f']
-			# Pad to always return microseconds.
-			s += "0" * (6 - len(s))
-			fraction = int(s)
-		elif group_key == 'A':
-			weekday = locale_time.f_weekday.index(found_dict['A'].lower())
-		elif group_key == 'a':
-			weekday = locale_time.a_weekday.index(found_dict['a'].lower())
-		elif group_key == 'w':
-			weekday = int(found_dict['w'])
+		elif key == 'M':
+			minute = int(val)
+		elif key == 'S':
+			second = int(val)
+		elif key == 'f':
+			if msec:
+				s = val
+				# Pad to always return microseconds.
+				s += "0" * (6 - len(s))
+				fraction = int(s)
+		elif key == 'A':
+			weekday = locale_time.f_weekday.index(val.lower())
+		elif key == 'a':
+			weekday = locale_time.a_weekday.index(val.lower())
+		elif key == 'w':
+			weekday = int(val)
 			if weekday == 0:
 				weekday = 6
 			else:
 				weekday -= 1
-		elif group_key == 'j':
-			julian = int(found_dict['j'])
-		elif group_key in ('U', 'W'):
-			week_of_year = int(found_dict[group_key])
-			if group_key == 'U':
+		elif key == 'j':
+			julian = int(val)
+		elif key in ('U', 'W'):
+			week_of_year = int(val)
+			if key == 'U':
 				# U starts week on Sunday.
 				week_of_year_start = 6
 			else:
 				# W starts week on Monday.
 				week_of_year_start = 0
-		elif group_key == 'z':
-			z = found_dict['z']
+		elif key == 'z':
+			z = val
 			if z == "Z":
 				tzoffset = 0
 			else:
@@ -227,7 +229,7 @@ def reGroupDictStrptime(found_dict):
 	# Actully create date
 	date_result =  datetime.datetime(
 		year, month, day, hour, minute, second, fraction)
-	if gmtoff:
+	if gmtoff is not None:
 		date_result = date_result - datetime.timedelta(seconds=gmtoff)
 
 	if date_result > now and assume_today:
@@ -240,7 +242,9 @@ def reGroupDictStrptime(found_dict):
 			year=year-1, month=month, day=day)
 
 	if gmtoff is not None:
-		return calendar.timegm(date_result.utctimetuple())
+		tm = calendar.timegm(date_result.utctimetuple())
 	else:
-		return time.mktime(date_result.timetuple())
-
+		tm = time.mktime(date_result.timetuple())
+	if msec:
+		tm += fraction/1000000.0
+	return tm
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 9616edbb..b862bbdb 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -202,6 +202,9 @@ class DateDetectorTest(LogCaptureTestCase):
 		dd = DateDetector()
 		dd.addDefaultTemplate()
 		for (debit, line, cnt) in (
+			# shortest distance to datetime should win:
+			("030324  0:03:59",            "some free text 030324  0:03:59 -- 2003-03-07 17:05:01 ...", 1),
+			# some free text with datetime:
 			("2003-03-07 17:05:01",        "some free text 2003-03-07 17:05:01 test ...", 15),
 			# distance collision detection (date from foreign input should not be found):
 			("030324  0:04:00",            "server mysqld[1000]: 030324  0:04:00 [Warning] Access denied ..."

From f56ff5f48b1075e59884e86a1a18332d98311c77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 12:11:06 +0200
Subject: [PATCH 225/623] optimized to better usage of the last time template
 (parse part of line at the same place as last time, if enclosed in the same
 boundaries) thereby follow rule "shortest distance to datetime should win",
 so possible collision causes search though all templates; speedup it a little
 bit (not possible collision if distance <= 1 or if line-begin anchoring, so
 break search if such template found)

---
 fail2ban/server/datedetector.py        | 132 +++++++++++++++++--------
 fail2ban/server/datetemplate.py        |  21 +++-
 fail2ban/tests/datedetectortestcase.py |  57 +++++++----
 3 files changed, 145 insertions(+), 65 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index e2c14d05..ba2b6377 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -26,7 +26,7 @@ import time
 
 from threading import Lock
 
-from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch
+from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -142,6 +142,10 @@ class DateDetectorTemplate(object):
 		# the last distance to date-match within the log file:
 		self.distance = 0x7fffffff
 
+	@property
+	def weight(self):
+		return self.hits * self.template.weight / max(1, self.distance)
+
 	def __getattr__(self, name):
 		""" Returns attribute of template (called for parameters not in slots)
 		"""
@@ -158,13 +162,14 @@ class DateDetector(object):
 	_defCache = DateDetectorCache()
 
 	def __init__(self):
-		self.__lock = Lock()
 		self.__templates = list()
 		self.__known_names = set()
 		# time the template was long unused (currently 300 == 5m):
 		self.__unusedTime = 300
-		# last known distance:
-		self.__lastDistance = 0
+		# last known distance (bypass one char collision) and end position:
+		self.__lastPos = 1, None
+		self.__lastEndPos = 0x7fffffff, None
+		self.__lastTemplIdx = 0x7fffffff
 		# first free place:
 		self.__firstUnused = 0
 
@@ -198,9 +203,8 @@ class DateDetector(object):
 	def addDefaultTemplate(self):
 		"""Add Fail2Ban's default set of date templates.
 		"""
-		with self.__lock:
-			for template in DateDetector._defCache.templates:
-				self._appendTemplate(template)
+		for template in DateDetector._defCache.templates:
+			self._appendTemplate(template)
 
 	@property
 	def templates(self):
@@ -226,29 +230,59 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
+		#logSys.log(logLevel, "try to match time for line: %.250s", line)
 		match = None
-		i = 0
-		found = None, 0x7fffffff, -1
-		with self.__lock:
+		# first try to use last template with same start/end position:
+		i = self.__lastTemplIdx
+		if i < len(self.__templates):
+			ddtempl = self.__templates[i]
+			template = ddtempl.template
+			distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
+			# check same boundaries left/right, otherwise possible collision/pattern switch:
+			if (line[distance-1:distance] == self.__lastPos[1] and 
+					line[endpos:endpos+1] == self.__lastEndPos[1]
+			):
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ... %s ...",
+						i, distance, endpos, line[distance:endpos])
+				match = template.matchDate(line, distance, endpos)
+				if match:
+					distance = match.start()
+					endpos = match.end()
+					# if different position, possible collision/pattern switch:
+					if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
+						logSys.log(logLevel, "  matched last time template #%02i", i)
+					else:
+						logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
+						match = None
+		# search template and better match:
+		if not match:
+			self.__lastTemplIdx = 0x7fffffff
+			logSys.log(logLevel, " search template ...")
+			found = None, 0x7fffffff, -1
+			i = 0
 			for ddtempl in self.__templates:
 				template = ddtempl.template
 				match = template.matchDate(line)
-				if match is not None:
-					distance = max(1, match.start() + 1)
+				if match:
+					distance = match.start()
+					endpos = match.end()
 					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "  matched time template #%r (at %r <= %r, %r) %s",
-							i, distance, ddtempl.distance, self.__lastDistance, template.name)
+						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
+							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
+					## if line-begin anchored - stop searching:
+					if template.flags & DateTemplate.LINE_BEGIN:
+						break
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
 					if (
-						(distance > ddtempl.distance or distance > self.__lastDistance) and
+						(distance > ddtempl.distance or distance > self.__lastPos[0]) and
 						len(self.__templates) > 1
 					):
-						if logSys.getEffectiveLevel() <= logLevel:
-							logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
+						logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
 						if distance < found[1]:
-							found = match, distance, i
+							found = match, distance, endpos, i
 						## search further:
 						match = None
 						i += 1
@@ -257,25 +291,29 @@ class DateDetector(object):
 					break
 				i += 1
 			# check other template was found (use this one with shortest distance):
-			if match is None and found[0]:
-				match, distance, i = found
+			if not match and found[0]:
+				match, distance, endpos, i = found
+				logSys.log(logLevel, "  use best time template #%02i", i)
 				ddtempl = self.__templates[i]
 				template = ddtempl.template
-			# we've winner, incr hits, set distance, usage, reorder, etc:
-			if match is not None:
-				ddtempl.hits += 1
-				self.__lastDistance = ddtempl.distance = distance
-				ddtempl.lastUsed = time.time()
-				if self.__firstUnused == i:
-					self.__firstUnused += 1
-				# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
-				if i:
-					logSys.log(logLevel, "  -> reorder template #%r, hits: %r", i, ddtempl.hits)
-					self._reorderTemplate(i)
-				# return tuple with match and template reference used for parsing:
-				return (match, template)
+		# we've winner, incr hits, set distance, usage, reorder, etc:
+		if match:
+			ddtempl.hits += 1
+			ddtempl.lastUsed = time.time()
+			ddtempl.distance = distance
+			if self.__firstUnused == i:
+				self.__firstUnused += 1
+			self.__lastPos = distance, line[distance-1:distance]
+			self.__lastEndPos = endpos, line[endpos:endpos+1]
+			# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
+			if i:
+				i = self._reorderTemplate(i)
+			self.__lastTemplIdx = i
+			# return tuple with match and template reference used for parsing:
+			return (match, template)
 
 		# not found:
+		logSys.log(logLevel, " no template.")
 		return (None, None)
 
 	def getTime(self, line, timeMatch=None):
@@ -311,7 +349,8 @@ class DateDetector(object):
 							date[0], date[1].group(), template.name)
 					return date
 			except ValueError:
-				return None
+				pass
+		return None
 
 	def _reorderTemplate(self, num):
 		"""Reorder template (bubble up) in template list if hits grows enough.
@@ -324,29 +363,36 @@ class DateDetector(object):
 		if num:
 			templates = self.__templates
 			ddtempl = templates[num]
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "  -> reorder template #%02i, hits: %r", num, ddtempl.hits)
 		  ## current hits and time the template was long unused:
 			untime = ddtempl.lastUsed - self.__unusedTime
-			weight = ddtempl.hits * ddtempl.template.weight / ddtempl.distance
+			weight = ddtempl.weight
 			## try to move faster (first if unused available, or half of part to current template position):
 			pos = self.__firstUnused if self.__firstUnused < num else num // 2
-			pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
+			pweight = templates[pos].weight
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
-			logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r, hits %r > %r",
-				num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "  -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r",
+					num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
 			if not pweight or weight > pweight or templates[pos].lastUsed < untime:
 				## if not larger (and target position recently used) - move slow (exact 1 position):
 				if weight <= pweight and templates[pos].lastUsed > untime:
 					pos = num-1
 					## if still smaller and template at position used, don't move:
-					pweight = templates[pos].hits * templates[pos].template.weight / templates[pos].distance
-					logSys.log(logLevel, "  -> compare template #%r & #%r, weight %r > %r, hits %r > %r",
-						num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
+					pweight = templates[pos].weight
+					if logSys.getEffectiveLevel() <= logLevel:
+						logSys.log(logLevel, "  -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r",
+							num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
 					if weight < pweight and templates[pos].lastUsed > untime:
 						return
 				del templates[num]
 				templates[pos:0] = [ddtempl]
 				## correct first unused:
-				if pos == self.__firstUnused:
+				while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits:
 					self.__firstUnused += 1
-				logSys.log(logLevel, "  -> moved template #%r -> #%r", num, pos)
+				if logSys.getEffectiveLevel() <= logLevel:
+					logSys.log(logLevel, "  -> moved template #%02i -> #%02i", num, pos)
+				return pos
+		return num
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 3d116b1e..d85c8f66 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -48,9 +48,14 @@ class DateTemplate(object):
 	regex
 	"""
 
+	LINE_BEGIN = 8
+	WORD_BEGIN = 2
+	WORD_END =   1
+
 	def __init__(self):
 		self.name = ""
 		self.weight = 1.0
+		self.flags = 0
 		self._regex = ""
 		self._cRegex = None
 
@@ -83,12 +88,14 @@ class DateTemplate(object):
 		regex = regex.strip()
 		# if word or line start boundary:
 		if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
+			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
 			regex = (r'(?=^|\b|\W)' if wordBegin != 'start' else r"(?:^|(?<=^\W)|(?<=^\W{2}))") + regex
 			self.name = ('{*WD-BEG}' if wordBegin != 'start' else '{^LN-BEG}') + self.name
 		# if word end boundary:
 		if wordEnd and not RE_NO_WRD_BOUND_END.search(regex):
+			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
-			self.name += ('{*WD-END}' if wordEnd else '')
+			self.name += '{*WD-END}'
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS.sub('', regex)
 		self._regex = regex
@@ -97,12 +104,18 @@ class DateTemplate(object):
 		"""Regex used to search for date.
 		""")
 
-	def matchDate(self, line):
-		"""Check if regex for date matches on a log line.
+	def _compileRegex(self):
+		"""Compile regex by first usage.
 		"""
 		if not self._cRegex:
 			self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE)
-		dateMatch = self._cRegex.search(line)
+
+	def matchDate(self, line, *args):
+		"""Check if regex for date matches on a log line.
+		"""
+		if not self._cRegex:
+			self._compileRegex()
+		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		return dateMatch
 
 	@abstractmethod
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index b862bbdb..a4c9b645 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -42,17 +42,20 @@ class DateDetectorTest(LogCaptureTestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		LogCaptureTestCase.setUp(self)
-		self.__old_eff_level = datedetector.logLevel
-		datedetector.logLevel = logSys.getEffectiveLevel()
 		setUpMyTime()
-		self.__datedetector = DateDetector()
-		self.__datedetector.addDefaultTemplate()
+		self.__datedetector = None
 
 	def tearDown(self):
 		"""Call after every test case."""
 		LogCaptureTestCase.tearDown(self)
-		datedetector.logLevel = self.__old_eff_level
 		tearDownMyTime()
+
+	@property
+	def datedetector(self):
+		if self.__datedetector is None:
+			self.__datedetector = DateDetector()
+			self.__datedetector.addDefaultTemplate()
+		return self.__datedetector
 	
 	def testGetEpochTime(self):
 		# correct epoch time, using all variants:
@@ -60,7 +63,7 @@ class DateDetectorTest(LogCaptureTestCase):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
 				date = date % dateUnix
 				log = date + " [sshd] error: PAM: Authentication failure"
-				datelog = self.__datedetector.getTime(log)
+				datelog = self.datedetector.getTime(log)
 				self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,))
 				( datelog, matchlog ) = datelog
 				self.assertEqual(int(datelog), dateUnix)
@@ -70,7 +73,7 @@ class DateDetectorTest(LogCaptureTestCase):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
 				date = date % dateUnix
 				log = date + " [sshd] error: PAM: Authentication failure"
-				datelog = self.__datedetector.getTime(log)
+				datelog = self.datedetector.getTime(log)
 				self.assertFalse(datelog)
 	
 	def testGetTime(self):
@@ -80,7 +83,7 @@ class DateDetectorTest(LogCaptureTestCase):
 		#      is not correctly determined atm, since year is not present
 		#      in the log entry.  Since this doesn't effect the operation
 		#      of fail2ban -- we just ignore incorrect day of the week
-		( datelog, matchlog ) = self.__datedetector.getTime(log)
+		( datelog, matchlog ) = self.datedetector.getTime(log)
 		self.assertEqual(datelog, dateUnix)
 		self.assertEqual(matchlog.group(), 'Jan 23 21:59:59')
 
@@ -140,7 +143,7 @@ class DateDetectorTest(LogCaptureTestCase):
 				if not bound and prefix == "word-boundary": continue
 				logSys.debug('  -- test %-5s for %r', should_match, log)
 				# with getTime:
-				logtime = self.__datedetector.getTime(log)
+				logtime = self.datedetector.getTime(log)
 				if should_match:
 					self.assertNotEqual(logtime, None,
 						"getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log))
@@ -152,8 +155,8 @@ class DateDetectorTest(LogCaptureTestCase):
 					self.assertEqual(logtime, None,
 						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
 				# with getTime(matchTime) - this combination used in filter:
-				(timeMatch, template) = matchTime = self.__datedetector.matchTime(log)
-				logtime = self.__datedetector.getTime(log, matchTime)
+				(timeMatch, template) = matchTime = self.datedetector.matchTime(log)
+				logtime = self.datedetector.getTime(log, matchTime)
 				logSys.debug('  -- found - %r', template.name if timeMatch else False)
 				if should_match:
 					self.assertNotEqual(logtime, None,
@@ -168,26 +171,26 @@ class DateDetectorTest(LogCaptureTestCase):
 				logSys.debug('  -- OK')
 
 	def testAllUniqueTemplateNames(self):
-		self.assertRaises(ValueError, self.__datedetector.appendTemplate,
-						  self.__datedetector.templates[0])
+		self.assertRaises(ValueError, self.datedetector.appendTemplate,
+						  self.datedetector.templates[0])
 
 	def testFullYearMatch_gh130(self):
 		# see https://github.com/fail2ban/fail2ban/pull/130
 		# yoh: unfortunately this test is not really effective to reproduce the
 		#      situation but left in place to assure consistent behavior
 		mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).timetuple())
-		logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
+		logdate = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
 		self.assertNotEqual(logdate, None)
 		( logTime, logMatch ) = logdate
 		self.assertEqual(logTime, mu)
 		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 		# confuse it with year being at the end
 		for i in xrange(10):
-			( logTime, logMatch ) =	self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
+			( logTime, logMatch ) =	self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
 			self.assertEqual(logTime, mu)
 			self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
 		# and now back to the original
-		( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
+		( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
 		self.assertEqual(logTime, mu)
 		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 
@@ -199,8 +202,7 @@ class DateDetectorTest(LogCaptureTestCase):
 			self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
 
 	def testAmbiguousInOrderedTemplates(self):
-		dd = DateDetector()
-		dd.addDefaultTemplate()
+		dd = self.datedetector
 		for (debit, line, cnt) in (
 			# shortest distance to datetime should win:
 			("030324  0:03:59",            "some free text 030324  0:03:59 -- 2003-03-07 17:05:01 ...", 1),
@@ -224,6 +226,25 @@ class DateDetectorTest(LogCaptureTestCase):
 				self.assertTrue(match)
 				self.assertEqual(match.group(), debit)
 
+	def testLowLevelLogging(self):
+		# test coverage for the deep (heavy) debug messages:
+		try:
+			self.__old_eff_level = datedetector.logLevel
+			if datedetector.logLevel < logSys.getEffectiveLevel()+1:
+				datedetector.logLevel = logSys.getEffectiveLevel()+1
+			dd = self.datedetector
+			i = 0
+			for (line, cnt) in (
+				("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 2),
+				("server mysqld[5906]: 051007 06:10:%02i 5907 [Warning] Access denied", 5),
+				("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 10),
+			):
+				for i in range(i, i+cnt+1):
+					logSys.debug('== test: %r', (line % i, cnt))
+					match, template = dd.matchTime(line % i)
+					self.assertTrue(match)
+		finally:
+			datedetector.logLevel = self.__old_eff_level
 
 iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
 

From ab0ac2111c4e123b7f685235be99f515938fd193 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 5 Oct 2016 19:34:21 +0200
Subject: [PATCH 226/623] added possibility to specify more precise default
 date pattern:   - `datepattern = {^LN-BEG}` - only line-begin anchored
 default patterns      (matches date only at begin of line, or with max
 distance up to 2 non-alphanumeric characters from line-begin);   -
 `datepattern = {*WD-BEG}` - only word-begin anchored default patterns;   -
 `datepattern = ^prefix{DATE}suffix` - exact specified default patterns (using
 prefix and suffix); common filter configs gets a more precise, line-begin
 anchored (datepattern = {^LN-BEG}) resp. custom anchoring default
 date-patterns;

---
 config/filter.d/3proxy.conf                   |  2 +
 config/filter.d/apache-common.conf            |  2 +
 config/filter.d/apache-pass.conf              |  4 --
 config/filter.d/assp.conf                     |  2 +
 config/filter.d/asterisk.conf                 |  1 +
 config/filter.d/common.conf                   |  3 ++
 config/filter.d/counter-strike.conf           |  2 -
 config/filter.d/courier-auth.conf             |  2 +
 config/filter.d/directadmin.conf              |  1 -
 config/filter.d/freeswitch.conf               |  2 +
 config/filter.d/kerio.conf                    |  2 -
 config/filter.d/monit.conf                    |  2 +-
 config/filter.d/murmur.conf                   |  3 +-
 config/filter.d/nginx-http-auth.conf          |  2 +
 config/filter.d/nginx-limit-req.conf          |  1 +
 config/filter.d/openhab.conf                  |  1 -
 config/filter.d/oracleims.conf                |  4 +-
 config/filter.d/selinux-common.conf           |  2 +
 config/filter.d/sogo-auth.conf                |  4 +-
 config/filter.d/squirrelmail.conf             |  2 -
 config/filter.d/sshd.conf                     |  4 +-
 fail2ban/client/fail2banregex.py              |  5 +-
 fail2ban/client/filterreader.py               | 17 ++++---
 fail2ban/server/datedetector.py               | 48 +++++++++++++------
 fail2ban/server/datetemplate.py               | 20 ++++++--
 fail2ban/server/filter.py                     | 35 ++++++++++----
 fail2ban/server/strptime.py                   |  1 +
 fail2ban/tests/files/filter.d/testcase01.conf |  1 -
 fail2ban/tests/servertestcase.py              |  2 +-
 29 files changed, 120 insertions(+), 57 deletions(-)

diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf
index 299c3a29..76c7573c 100644
--- a/config/filter.d/3proxy.conf
+++ b/config/filter.d/3proxy.conf
@@ -9,6 +9,8 @@ failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ <HOST>:\d+ [\d.]+:\d+ \d+ \d+ \d+\s
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 # http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are
 # all authentication problems (%E field)
diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf
index c843c92e..3d1f902b 100644
--- a/config/filter.d/apache-common.conf
+++ b/config/filter.d/apache-common.conf
@@ -10,6 +10,8 @@ after = apache-common.local
 
 _apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
 
+datepattern = {^LN-BEG}
+
 # Common prefix for [error] apache messages which also would include <HOST>
 # Depending on the version it could be
 # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]
diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf
index dd00f953..7bdff9c0 100644
--- a/config/filter.d/apache-pass.conf
+++ b/config/filter.d/apache-pass.conf
@@ -3,10 +3,6 @@
 #
 # The knocking request must have a referer.
 
-[INCLUDES]
-
-before = apache-common.conf
-
 [Definition]
 
 failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$
diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf
index 278e25cb..8d6367c9 100644
--- a/config/filter.d/assp.conf
+++ b/config/filter.d/assp.conf
@@ -20,6 +20,8 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 # V1 Examples matches:
 #   Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);
diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 79bd1ff2..df55d288 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -31,6 +31,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
 
 ignoreregex =
 
+datepattern = {^LN-BEG}
 
 # Author: Xavier Devlamynck / Daniel Black
 #
diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf
index 586f428a..a8cba188 100644
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@@ -61,4 +61,7 @@ __prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostn
 # pam_ldap
 __pam_auth = pam_unix
 
+# standardly all formats using prefix have line-begin anchored date:
+datepattern = {^LN-BEG}
+
 # Author: Yaroslav Halchenko
diff --git a/config/filter.d/counter-strike.conf b/config/filter.d/counter-strike.conf
index a896b5ca..294927bc 100644
--- a/config/filter.d/counter-strike.conf
+++ b/config/filter.d/counter-strike.conf
@@ -8,8 +8,6 @@ failregex = ^: Bad Rcon: "rcon \d+ "\S+"  sv_contact ".*?"" from "<HOST>:\d+"$
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S
 
 
diff --git a/config/filter.d/courier-auth.conf b/config/filter.d/courier-auth.conf
index 1170a63a..87ee55b4 100644
--- a/config/filter.d/courier-auth.conf
+++ b/config/filter.d/courier-auth.conf
@@ -15,5 +15,7 @@ failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # Author: Christoph Haas
 # Modified by: Cyril Jaquier
diff --git a/config/filter.d/directadmin.conf b/config/filter.d/directadmin.conf
index 7622e548..87c7802f 100644
--- a/config/filter.d/directadmin.conf
+++ b/config/filter.d/directadmin.conf
@@ -13,7 +13,6 @@ failregex = ^: \'<HOST>\' \d{1,3} failed login attempt(s)?. \s*
 
 ignoreregex = 
 
-[Init]
 datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S
 
 #
diff --git a/config/filter.d/freeswitch.conf b/config/filter.d/freeswitch.conf
index 29940240..4759fbed 100644
--- a/config/filter.d/freeswitch.conf
+++ b/config/filter.d/freeswitch.conf
@@ -26,6 +26,8 @@ failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challe
 
 ignoreregex =
 
+datepattern = {^LN-BEG}
+
 # Author: Rupa SChomaker, soapee01, Daniel Black
 # https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban
 # Thanks to Jim on mailing list of samples and guidance
diff --git a/config/filter.d/kerio.conf b/config/filter.d/kerio.conf
index 313c9b36..e0d94753 100644
--- a/config/filter.d/kerio.conf
+++ b/config/filter.d/kerio.conf
@@ -9,8 +9,6 @@ failregex = ^ SMTP Spam attack detected from <HOST>,
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
 
 # DEV NOTES:
diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf
index 599bd363..b652a1f4 100644
--- a/config/filter.d/monit.conf
+++ b/config/filter.d/monit.conf
@@ -13,7 +13,7 @@ before = common.conf
 _daemon = monit
 
 # Regexp for previous (accessing monit httpd) and new (access denied) versions
-failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
+failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
             ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
 
 # Ignore login with empty user (first connect, no user specified)
diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf
index 3775a9d2..507bbd2f 100644
--- a/config/filter.d/murmur.conf
+++ b/config/filter.d/murmur.conf
@@ -15,13 +15,14 @@ _daemon = murmurd
 #      variable in your server config file (murmur.ini / mumble-server.ini).
 _usernameregex = [^>]+
 
-_prefix = <W>[\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
+_prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
 
 failregex = ^%(_prefix)s Invalid server password$
             ^%(_prefix)s Wrong certificate or password for existing user$
 
 ignoreregex =
 
+datepattern = ^<W>{DATE}
 
 # DEV Notes:
 #
diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf
index a689f66a..e45a7f49 100644
--- a/config/filter.d/nginx-http-auth.conf
+++ b/config/filter.d/nginx-http-auth.conf
@@ -8,6 +8,8 @@ failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not f
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV NOTES:
 # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files
 # Extensive search of all nginx auth failures not done yet.
diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf
index 13cf75c2..c283849c 100644
--- a/config/filter.d/nginx-limit-req.conf
+++ b/config/filter.d/nginx-limit-req.conf
@@ -43,3 +43,4 @@ failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by z
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
diff --git a/config/filter.d/openhab.conf b/config/filter.d/openhab.conf
index 83857c7a..f6b96339 100644
--- a/config/filter.d/openhab.conf
+++ b/config/filter.d/openhab.conf
@@ -9,7 +9,6 @@
 [Definition] 
 failregex = ^<HOST>\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$
 
-[Init]
 datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z
 
 
diff --git a/config/filter.d/oracleims.conf b/config/filter.d/oracleims.conf
index e951ff5d..7d75c322 100644
--- a/config/filter.d/oracleims.conf
+++ b/config/filter.d/oracleims.conf
@@ -52,10 +52,12 @@ before = common.conf
 # Note that you MUST have LOG_FORMAT=4 for this to work!
 #
 
-failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
+failregex = tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
 # Values:  TEXT
 #
 ignoreregex =
+
+datepattern = ^<co ts="{DATE}"\s+
diff --git a/config/filter.d/selinux-common.conf b/config/filter.d/selinux-common.conf
index 7269e8f7..b3e0ae4f 100644
--- a/config/filter.d/selinux-common.conf
+++ b/config/filter.d/selinux-common.conf
@@ -18,4 +18,6 @@ failregex = ^type=%(_type)s msg=audit\(:\d+\): (user )?pid=\d+ uid=%(_uid)s auid
 
 ignoreregex =
 
+datepattern = EPOCH
+
 # Author: Daniel Black
diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf
index d56c94f7..1a3d5292 100644
--- a/config/filter.d/sogo-auth.conf
+++ b/config/filter.d/sogo-auth.conf
@@ -6,7 +6,9 @@
 
 failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d*  grace: -?\d*  expire: -?\d*  bound: -?\d*)?\s*$
 
-ignoreregex = 
+ignoreregex = "^<ADDR>"
+
+datepattern = {^LN-BEG}
 
 # 
 # DEV Notes:
diff --git a/config/filter.d/squirrelmail.conf b/config/filter.d/squirrelmail.conf
index af0c38e7..31e922e6 100644
--- a/config/filter.d/squirrelmail.conf
+++ b/config/filter.d/squirrelmail.conf
@@ -5,8 +5,6 @@ failregex = ^ \[LOGIN_ERROR\].*from <HOST>: Unknown user or password incorrect\.
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S
 
 # DEV NOTES:
diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index eeb1518e..c1c8eacf 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -38,13 +38,13 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro
 
 ignoreregex = 
 
-[Init]
-
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 10
 
 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 #
 #   "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 6680b097..bf49e7a6 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -474,8 +474,9 @@ class Fail2banRegex(object):
 				if self._verbose or template.hits:
 					out.append("[%d] %s" % (template.hits, template.name))
 					if self._verbose_date:
-						out.append("    # weight: %3s, pattern: %s" % (
-							template.weight, getattr(template, 'pattern', ''),))
+						out.append("    # weight: %.3f (%.3f), pattern: %s" % (
+							template.weight, template.template.weight,
+							getattr(template, 'pattern', ''),))
 						out.append("    # regex:   %s" % (getattr(template, 'regex', ''),))
 			pprint_list(out, "[# of hits] date format")
 
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index 8b30f914..9bd94fd6 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -40,6 +40,9 @@ class FilterReader(DefinitionInitConfigReader):
 	_configOpts = {
 		"ignoreregex": ["string", None],
 		"failregex": ["string", ""],
+		"maxlines": ["int", None],
+		"datepattern": ["string", None],
+		"journalmatch": ["string", None],
 	}
 
 	def setFile(self, fileName):
@@ -74,16 +77,16 @@ class FilterReader(DefinitionInitConfigReader):
 					stream.append(["multi-set", self._jailName, "add" + opt, multi])
 				elif len(multi):
 					stream.append(["set", self._jailName, "add" + opt, multi[0]])
-		if self._initOpts:
-			if 'maxlines' in self._initOpts:
+			elif opt == 'maxlines':
 				# We warn when multiline regex is used without maxlines > 1
 				# therefore keep sure we set this option first.
-				stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]])
-			if 'datepattern' in self._initOpts:
-				stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]])
+				stream.insert(0, ["set", self._jailName, "maxlines", value])
+			elif opt == 'datepattern':
+				stream.append(["set", self._jailName, "datepattern", value])
 			# Do not send a command if the match is empty.
-			if self._initOpts.get("journalmatch", '') != '':
-				for match in self._initOpts["journalmatch"].split("\n"):
+			elif opt == 'journalmatch':
+				for match in value.split("\n"):
+					if match == '': continue
 					stream.append(
 						["set", self._jailName, "addjournalmatch"] +
                         shlex.split(match))
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index ba2b6377..0aac4337 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -26,7 +26,7 @@ import time
 
 from threading import Lock
 
-from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -34,6 +34,8 @@ logSys = getLogger(__name__)
 
 logLevel = 6
 
+RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE)
+
 
 class DateDetectorCache(object):
 	"""Implements the caching of the default templates list.
@@ -54,15 +56,18 @@ class DateDetectorCache(object):
 			self._addDefaultTemplate()
 			return self.__templates
 
-	def _cacheTemplate(self, template):
+	def _cacheTemplate(self, template, lineBeginOnly=False):
 		"""Cache Fail2Ban's default template.
 
 		"""
 		if isinstance(template, str):
 			# exact given template with word benin-end boundary:
-			template = DatePatternRegex(template)
+			if not lineBeginOnly:
+				template = DatePatternRegex(template)
+			else:
+				template = DatePatternRegex(template, wordBegin='start')
 		# additional template, that prefers datetime at start of a line (safety+performance feature):
-		if hasattr(template, 'regex'):
+		if not lineBeginOnly and hasattr(template, 'regex'):
 			template2 = copy.copy(template)
 			regex = getattr(template, 'pattern', template.regex)
 			template2.setRegex(regex, wordBegin='start', wordEnd=True)
@@ -90,6 +95,7 @@ class DateDetectorCache(object):
 		# prefixed with optional time zone (monit):
 		# PDT Apr 16 21:05:29
 		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
@@ -114,17 +120,18 @@ class DateDetectorCache(object):
 		# TAI64N
 		self._cacheTemplate(DateTai64n())
 		# Epoch
+		self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
 		self._cacheTemplate(DateEpoch())
 		# Only time information in the log
-		self._cacheTemplate("^%H:%M:%S")
+		self._cacheTemplate("%H:%M:%S", lineBeginOnly=True)
 		# <09/16/08@05:03:30>
-		self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>")
+		self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True)
 		# MySQL: 130322 11:46:11
 		self._cacheTemplate("%Exy%Exm%Exd  ?%H:%M:%S")
 		# Apache Tomcat
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
+		self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True)
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
 
@@ -172,6 +179,8 @@ class DateDetector(object):
 		self.__lastTemplIdx = 0x7fffffff
 		# first free place:
 		self.__firstUnused = 0
+		# pre-match pattern:
+		self.__preMatch = None
 
 	def _appendTemplate(self, template):
 		name = template.name
@@ -200,10 +209,18 @@ class DateDetector(object):
 			template = DatePatternRegex(template)
 		self._appendTemplate(template)
 
-	def addDefaultTemplate(self):
+	def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
 		"""Add Fail2Ban's default set of date templates.
 		"""
 		for template in DateDetector._defCache.templates:
+			# filter if specified:
+			if filterTemplate is not None and not filterTemplate(template): continue
+			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
+			if preMatch is not None:
+				regex = getattr(template, 'pattern', template.regex)
+				template = copy.copy(template)
+				template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
+			# append date detector template:
 			self._appendTemplate(template)
 
 	@property
@@ -230,7 +247,7 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
-		#logSys.log(logLevel, "try to match time for line: %.250s", line)
+		#logSys.log(logLevel, "try to match time for line: %.120s", line)
 		match = None
 		# first try to use last template with same start/end position:
 		i = self.__lastTemplIdx
@@ -238,13 +255,16 @@ class DateDetector(object):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
 			distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
+			if logSys.getEffectiveLevel() <= logLevel-1:
+				logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
+					i, distance, endpos, 
+					line[distance-1:distance], self.__lastPos[1],
+					line[distance:endpos],
+					line[endpos:endpos+1], self.__lastEndPos[1])
 			# check same boundaries left/right, otherwise possible collision/pattern switch:
 			if (line[distance-1:distance] == self.__lastPos[1] and 
 					line[endpos:endpos+1] == self.__lastEndPos[1]
 			):
-				if logSys.getEffectiveLevel() <= logLevel-1:
-					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ... %s ...",
-						i, distance, endpos, line[distance:endpos])
 				match = template.matchDate(line, distance, endpos)
 				if match:
 					distance = match.start()
@@ -270,8 +290,8 @@ class DateDetector(object):
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
 							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
-					## if line-begin anchored - stop searching:
-					if template.flags & DateTemplate.LINE_BEGIN:
+					## if line-begin/end anchored - stop searching:
+					if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 						break
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index d85c8f66..fcd51f99 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -33,9 +33,12 @@ from ..helpers import getLogger
 logSys = getLogger(__name__)
 
 RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
-RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$|\*\*)$')
+RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
 RE_DEL_WRD_BOUNDS =   re.compile(r'^\*\*|(?<!\\)\*\*$')
 
+RE_LINE_BOUND_BEG = re.compile(r'^(?:\^|\(\?:\^(?!\|))')
+RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
+
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -49,6 +52,7 @@ class DateTemplate(object):
 	"""
 
 	LINE_BEGIN = 8
+	LINE_END =   4
 	WORD_BEGIN = 2
 	WORD_END =   1
 
@@ -86,6 +90,7 @@ class DateTemplate(object):
 			If regular expression fails to compile
 		"""
 		regex = regex.strip()
+		self.flags = 0
 		# if word or line start boundary:
 		if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
@@ -96,9 +101,12 @@ class DateTemplate(object):
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
 			self.name += '{*WD-END}'
+		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
+		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS.sub('', regex)
 		self._regex = regex
+		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
 		"""Regex used to search for date.
@@ -151,11 +159,15 @@ class DateEpoch(DateTemplate):
 	regex
 	"""
 
-	def __init__(self):
+	def __init__(self, lineBeginOnly=False):
 		DateTemplate.__init__(self)
 		self.name = "Epoch"
-		self.setRegex(r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))",
-			wordBegin=False) ;# already line begin resp. word begin anchored
+		if not lineBeginOnly:
+			regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
+			self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
+		else:
+			regex = r"(?P<square>(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))"
+			self.setRegex(regex, wordBegin='start', wordEnd=True)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 9b1aefd6..cbf59fcb 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -35,7 +35,7 @@ from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
-from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n
+from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
@@ -257,14 +257,29 @@ class Filter(JailThread):
 		if pattern is None:
 			self.dateDetector = None
 			return
-		elif pattern.upper() == "EPOCH":
-			template = DateEpoch()
-			template.name = "Epoch"
-		elif pattern.upper() == "TAI64N":
-			template = DateTai64n()
-			template.name = "TAI64N"
 		else:
-			template = DatePatternRegex(pattern)
+			key = pattern.upper()
+			if key == "EPOCH":
+				template = DateEpoch()
+				template.name = "Epoch"
+			elif key == "TAI64N":
+				template = DateTai64n()
+				template.name = "TAI64N"
+			elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
+				self.dateDetector = DateDetector()
+				flt = \
+					lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
+					lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
+					None
+				self.dateDetector.addDefaultTemplate(flt)
+				return
+			elif "{DATE}" in key:
+				self.dateDetector = DateDetector()
+				self.dateDetector.addDefaultTemplate(
+					lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
+				return
+			else:
+				template = DatePatternRegex(pattern)
 		self.dateDetector = DateDetector()
 		self.dateDetector.appendTemplate(template)
 		logSys.info("  date pattern `%r`: `%s`",
@@ -280,9 +295,9 @@ class Filter(JailThread):
 	def getDatePattern(self):
 		if self.dateDetector is not None:
 			templates = self.dateDetector.templates
-			if len(templates) > 1:
+			if len(templates) > 2:
 				return None, "Default Detectors"
-			elif len(templates) == 1:
+			elif len(templates):
 				if hasattr(templates[0], "pattern"):
 					pattern =  templates[0].pattern
 				else:
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 85045ed8..da00a57e 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -40,6 +40,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 
 #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
 #timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
+timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
 timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
 
 # Extend build-in TimeRE with some exact patterns
diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf
index 0dbbbd2a..b3700d0b 100644
--- a/fail2ban/tests/files/filter.d/testcase01.conf
+++ b/fail2ban/tests/files/filter.d/testcase01.conf
@@ -33,7 +33,6 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro
 #
 ignoreregex = ^.+ john from host 192.168.1.1\s*$
 
-[Init]
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 1
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index d4320257..cd7ce5c9 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1107,7 +1107,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# (we don't use it in this test at all):
 					elif unittest.F2B.fast and (
 						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
-					):
+					): # pragma: no cover
 						cmd[0] = "set"
 						cmd[3] = "DUMMY-REGEX <HOST>"
 					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:

From e735f8f56835abeda6135c2088b54753193de7c9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 6 Oct 2016 15:27:49 +0200
Subject: [PATCH 227/623] default non-unicode and case-sensitive matching (by
 pattern templates automatically add `(?iu)` for "ignore case" and "unicode"
 if expected)

---
 fail2ban/server/datedetector.py        | 47 ++++++++++++++------------
 fail2ban/server/datetemplate.py        | 25 ++++++++++----
 fail2ban/tests/datedetectortestcase.py | 45 +++++++++++++++++++++---
 3 files changed, 85 insertions(+), 32 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 0aac4337..f256a425 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -254,27 +254,32 @@ class DateDetector(object):
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
-			distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
-			if logSys.getEffectiveLevel() <= logLevel-1:
-				logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
-					i, distance, endpos, 
-					line[distance-1:distance], self.__lastPos[1],
-					line[distance:endpos],
-					line[endpos:endpos+1], self.__lastEndPos[1])
-			# check same boundaries left/right, otherwise possible collision/pattern switch:
-			if (line[distance-1:distance] == self.__lastPos[1] and 
-					line[endpos:endpos+1] == self.__lastEndPos[1]
-			):
-				match = template.matchDate(line, distance, endpos)
-				if match:
-					distance = match.start()
-					endpos = match.end()
-					# if different position, possible collision/pattern switch:
-					if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
-						logSys.log(logLevel, "  matched last time template #%02i", i)
-					else:
-						logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
-						match = None
+			if template.flags & DateTemplate.LINE_BEGIN:
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
+					match = template.matchDate(line)
+			else:
+				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
+						i, distance, endpos, 
+						line[distance-1:distance], self.__lastPos[1],
+						line[distance:endpos],
+						line[endpos:endpos+1], self.__lastEndPos[1])
+				# check same boundaries left/right, otherwise possible collision/pattern switch:
+				if (line[distance-1:distance] == self.__lastPos[1] and 
+						line[endpos:endpos+1] == self.__lastEndPos[1]
+				):
+					match = template.matchDate(line, distance, endpos)
+			if match:
+				distance = match.start()
+				endpos = match.end()
+				# if different position, possible collision/pattern switch:
+				if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
+					logSys.log(logLevel, "  matched last time template #%02i", i)
+				else:
+					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
+					match = None
 		# search template and better match:
 		if not match:
 			self.__lastTemplIdx = 0x7fffffff
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index fcd51f99..9524e373 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -32,13 +32,18 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
-RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)')
 RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
-RE_DEL_WRD_BOUNDS =   re.compile(r'^\*\*|(?<!\\)\*\*$')
+RE_DEL_WRD_BOUNDS = ( re.compile(r'^(?:\(\?\w+\))?\*\*|(?<!\\)\*\*()$'), 
+	                    lambda m: m.group().replace('**', '') )
 
-RE_LINE_BOUND_BEG = re.compile(r'^(?:\^|\(\?:\^(?!\|))')
+RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
 RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
+RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
+
+_Templ_RECache = {}
+
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -104,7 +109,7 @@ class DateTemplate(object):
 		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
 		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
-		regex = RE_DEL_WRD_BOUNDS.sub('', regex)
+		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
 		self._cRegex = None
 
@@ -116,7 +121,11 @@ class DateTemplate(object):
 		"""Compile regex by first usage.
 		"""
 		if not self._cRegex:
-			self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE)
+			try:
+				self._cRegex = re.compile(self.regex)
+			except Exception as e:
+				logSys.error('Compile %r failed, expression %r', self.name, self.regex)
+				raise e
 
 	def matchDate(self, line, *args):
 		"""Check if regex for date matches on a log line.
@@ -235,7 +244,11 @@ class DatePatternRegex(DateTemplate):
 		self._pattern = pattern
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
 		self.name = fmt % self._patternName
-		super(DatePatternRegex, self).setRegex(fmt % timeRE, wordBegin, wordEnd)
+		regex = fmt % timeRE
+		# if expected add (?iu) for "ignore case" and "unicode":
+		if RE_ALPHA_PATTERN.search(pattern):
+			regex = r'(?iu)' + regex
+		super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index a4c9b645..6bf92582 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -195,11 +195,46 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 
 	def testDateTemplate(self):
-			t = DateTemplate()
-			t.setRegex('^a{3,5}b?c*$')
-			self.assertEqual(t.getRegex(), '^a{3,5}b?c*$')
-			self.assertRaises(Exception, t.getDate, '')
-			self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+		t = DateTemplate()
+		t.setRegex('^a{3,5}b?c*$')
+		self.assertEqual(t.regex, '^a{3,5}b?c*$')
+		self.assertRaises(Exception, t.getDate, '')
+		self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+
+		## no word boundaries left and right:
+		t = DatePatternRegex()
+		t.pattern = '(?iu)**time:%ExY%Exm%ExdT%ExH%ExM%ExS**'
+		# ** was removed from end-regex:
+		self.assertFalse('**' in t.regex)
+		# match date:
+		dt = 'TIME:20050102T010203'
+		self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt)
+		self.assertEqual(t.matchDate(dt).group(), dt)
+		# wrong year (for exact %ExY):
+		dt = 'TIME:50050102T010203'
+		self.assertFalse(t.matchDate(dt))
+
+		## start boundary left and word boundary right:
+		t = DatePatternRegex()
+		t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS'
+		self.assertTrue('^' in t.regex)
+		# try match date:
+		dt = 'time:20050102T010203'
+		self.assertFalse(t.matchDate('X' + dt))
+		self.assertFalse(t.matchDate(dt + 'X'))
+		self.assertEqual(t.matchDate('##' + dt + '...').group(), dt)
+		self.assertEqual(t.matchDate(dt).group(), dt)
+		# case sensitive:
+		dt = 'TIME:20050102T010203'
+		self.assertFalse(t.matchDate(dt))
+
+		## auto-switching "ignore case" and "unicode"
+		t = DatePatternRegex()
+		t.pattern = '^%Y %b %d'
+		self.assertTrue('(?iu)' in t.regex)
+		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt)
 
 	def testAmbiguousInOrderedTemplates(self):
 		dd = self.datedetector

From 0bed91b3c2be09df5a066273680e1164bce3f9c2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 6 Oct 2016 15:39:38 +0200
Subject: [PATCH 228/623] speedup SeekToTime test cases using exact date
 pattern...

---
 fail2ban/tests/filtertestcase.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 6c1a637c..ea996072 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -476,6 +476,8 @@ class LogFileFilterPoll(unittest.TestCase):
 		self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME))
 
 	def testSeekToTimeSmallFile(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 		fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log')
 		time = 1417512352
 		f = open(fname, 'w')
@@ -560,6 +562,8 @@ class LogFileFilterPoll(unittest.TestCase):
 			_killfile(f, fname)
 
 	def testSeekToTimeLargeFile(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 		fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log')
 		time = 1417512352
 		f = open(fname, 'w')

From bd1eb70c52f078fea65d1688915487a5dea1f3c7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 6 Oct 2016 16:44:07 +0200
Subject: [PATCH 229/623] speedup template first time selection through
 pre-sorted template list by template hits

---
 fail2ban/server/datedetector.py | 15 +++++++++------
 fail2ban/server/datetemplate.py |  3 +++
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index f256a425..7cbc0f5c 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -88,14 +88,9 @@ class DateDetectorCache(object):
 		# simple date: 2005/01/23 21:59:59 
 		# custom for syslog-ng 2006.12.21 06:43:20
 		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
-		# 20050123T215959, 20050123 215959
-		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		# prefixed with optional time zone (monit):
-		# PDT Apr 16 21:05:29
 		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
-		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
@@ -132,6 +127,12 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
 		self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True)
+		# 20050123T215959, 20050123 215959
+		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
+		# prefixed with optional named time zone (monit):
+		# PDT Apr 16 21:05:29
+		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		#
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
 
@@ -212,7 +213,9 @@ class DateDetector(object):
 	def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
 		"""Add Fail2Ban's default set of date templates.
 		"""
-		for template in DateDetector._defCache.templates:
+		for template in sorted(DateDetector._defCache.templates,
+			lambda a,b: b.hits - a.hits
+		):
 			# filter if specified:
 			if filterTemplate is not None and not filterTemplate(template): continue
 			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 9524e373..70a4b1b5 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -65,6 +65,7 @@ class DateTemplate(object):
 		self.name = ""
 		self.weight = 1.0
 		self.flags = 0
+		self.hits = 0
 		self._regex = ""
 		self._cRegex = None
 
@@ -133,6 +134,8 @@ class DateTemplate(object):
 		if not self._cRegex:
 			self._compileRegex()
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
+		if dateMatch:
+			self.hits += 1
 		return dateMatch
 
 	@abstractmethod

From ae7297e16b0188953bcdd18967bd22982b7fbfbb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 7 Oct 2016 14:57:45 +0200
Subject: [PATCH 230/623] more precise date template handling (WARNING: this
 commit creates possible incompatibilities):   - datedetector rewritten more
 strict as earlier;   - default templates can be specified exacter using
 prefix/suffix syntax (via `datepattern`);   - more as one date pattern can be
 specified using option `datepattern` now (new-line separated);   - some
 default options like `datepattern` can be specified directly in section
 `[Definition]`, that avoids contrary usage of unnecessarily `[Init]` section,
 because of performance (each extra section costs time);   - option
 `datepattern` can be specified in jail also (jails without filters);   - if
 first group specified, only this will be cut out from search log-line (e. g.:
 `^date:[({DATE})]` will cut out only datetime match pattern, and leaves
 `date:[] failure ip...` for searching in filter);   - faster match and fewer
 searching of appropriate templates (DateDetector.matchTime calls rarer
 DateTemplate.matchDate now);   - standard filters extended with exact
 prefixed or anchored date templates;

template cache introduced (in opposition to default template cache, holds custom templates cached by pattern for possible common usage of same template/regex);
---
 config/filter.d/apache-badbots.conf           |  3 +
 config/filter.d/apache-fakegooglebot.conf     |  2 +
 config/filter.d/apache-pass.conf              |  3 +
 config/filter.d/assp.conf                     |  3 +-
 config/filter.d/dovecot.conf                  |  5 +-
 config/filter.d/ejabberd-auth.conf            |  4 +-
 config/filter.d/guacamole.conf                |  5 +-
 config/filter.d/nginx-botsearch.conf          |  3 +
 config/filter.d/nsd.conf                      |  3 +
 config/filter.d/php-url-fopen.conf            |  3 +
 config/filter.d/portsentry.conf               |  3 +
 config/filter.d/sogo-auth.conf                |  5 +-
 config/filter.d/squid.conf                    |  3 +
 config/filter.d/tine20.conf                   |  3 +
 fail2ban/client/jailreader.py                 | 17 +---
 fail2ban/server/datedetector.py               | 99 ++++++++++++++-----
 fail2ban/server/datetemplate.py               | 59 +++++++----
 fail2ban/server/filter.py                     | 46 +++------
 fail2ban/server/strptime.py                   |  3 -
 fail2ban/server/transmitter.py                |  2 +-
 fail2ban/tests/action_d/test_badips.py        |  1 +
 fail2ban/tests/action_d/test_smtp.py          |  1 +
 fail2ban/tests/banmanagertestcase.py          |  2 +
 fail2ban/tests/clientbeautifiertestcase.py    |  1 +
 fail2ban/tests/clientreadertestcase.py        |  1 +
 .../config/filter.d/zzz-generic-example.conf  |  5 +
 fail2ban/tests/datedetectortestcase.py        | 59 +++++------
 fail2ban/tests/fail2banclienttestcase.py      |  1 +
 fail2ban/tests/fail2banregextestcase.py       |  9 ++
 fail2ban/tests/failmanagertestcase.py         |  1 +
 fail2ban/tests/files/logs/zzz-generic-example |  8 +-
 fail2ban/tests/filtertestcase.py              | 24 +++++
 fail2ban/tests/misctestcase.py                |  1 +
 fail2ban/tests/samplestestcase.py             |  1 +
 fail2ban/tests/servertestcase.py              |  8 +-
 fail2ban/tests/sockettestcase.py              |  1 +
 fail2ban/tests/utils.py                       | 18 +++-
 37 files changed, 280 insertions(+), 136 deletions(-)

diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf
index 48b30666..f42aa159 100644
--- a/config/filter.d/apache-badbots.conf
+++ b/config/filter.d/apache-badbots.conf
@@ -14,6 +14,9 @@ failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 # DEV Notes:
 # List of bad bots fetched from http://www.user-agents.org
 # Generated on Thu Nov  7 14:23:35 PST 2013 by files/gen_badbots.
diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf
index b8a73504..729410ad 100644
--- a/config/filter.d/apache-fakegooglebot.conf
+++ b/config/filter.d/apache-fakegooglebot.conf
@@ -6,6 +6,8 @@ failregex = ^<HOST> .*Googlebot.*$
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 #
diff --git a/config/filter.d/apache-pass.conf b/config/filter.d/apache-pass.conf
index 7bdff9c0..3cab87b0 100644
--- a/config/filter.d/apache-pass.conf
+++ b/config/filter.d/apache-pass.conf
@@ -9,6 +9,9 @@ failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 [Init]
 
 knocking_url = /knocking/
diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf
index 8d6367c9..7cca81d2 100644
--- a/config/filter.d/assp.conf
+++ b/config/filter.d/assp.conf
@@ -20,7 +20,8 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
 
 ignoreregex = 
 
-datepattern = {^LN-BEG}
+datepattern = {^LN-BEG}%%b-%%d-%%Exy %%H:%%M:%%S
+              {^LN-BEG}
 
 # DEV Notes:
 # V1 Examples matches:
diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index 136a3947..6f8510fc 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -17,10 +17,11 @@ failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentica
 
 ignoreregex = 
 
-[Init]
-
 journalmatch = _SYSTEMD_UNIT=dovecot.service
 
+datepattern = {^LN-BEG}TAI64N
+              {^LN-BEG}
+
 # DEV Notes:
 # * the first regex is essentially a copy of pam-generic.conf
 # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016)
diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf
index 512fdb7c..edd87463 100644
--- a/config/filter.d/ejabberd-auth.conf
+++ b/config/filter.d/ejabberd-auth.conf
@@ -25,8 +25,6 @@ failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\
 #
 ignoreregex = 
 
-[Init]
-
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
 
@@ -35,3 +33,5 @@ maxlines = 2
 # Values:  TEXT
 #
 journalmatch = 
+
+datepattern = ^(?:=[^=]+={3,} )?({DATE})
diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf
index 49cecc5a..09b4e7b0 100644
--- a/config/filter.d/guacamole.conf
+++ b/config/filter.d/guacamole.conf
@@ -17,6 +17,9 @@ failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" fa
 #
 ignoreregex = 
 
-[Init]
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
+
+datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
+              ^WARNING:()**
+              {^LN-BEG}
\ No newline at end of file
diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf
index 6853e1e8..0be895b2 100644
--- a/config/filter.d/nginx-botsearch.conf
+++ b/config/filter.d/nginx-botsearch.conf
@@ -13,6 +13,9 @@ failregex = ^<HOST> \- \S+ \[\] \"(GET|POST|HEAD) \/<block> \S+\" 404 .+$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 # Based on apache-botsearch filter
diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf
index 8f32f7be..bfd99544 100644
--- a/config/filter.d/nsd.conf
+++ b/config/filter.d/nsd.conf
@@ -26,3 +26,6 @@ failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
              ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =
+
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
\ No newline at end of file
diff --git a/config/filter.d/php-url-fopen.conf b/config/filter.d/php-url-fopen.conf
index 87bd04c8..a7957c9d 100644
--- a/config/filter.d/php-url-fopen.conf
+++ b/config/filter.d/php-url-fopen.conf
@@ -18,3 +18,6 @@ ignoreregex =
 # http://blogs.buanzo.com.ar/2009/04/fail2ban-filter-for-php-injection-attacks.html#comment-1489
 #
 # Author: Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
+
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
diff --git a/config/filter.d/portsentry.conf b/config/filter.d/portsentry.conf
index 27dca9b4..35ca2a3d 100644
--- a/config/filter.d/portsentry.conf
+++ b/config/filter.d/portsentry.conf
@@ -8,5 +8,8 @@ failregex = \/<HOST> Port\: [0-9]+ (TCP|UDP) Blocked$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Pacop <pacoparu@gmail.com>
 
diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf
index 1a3d5292..48221dc0 100644
--- a/config/filter.d/sogo-auth.conf
+++ b/config/filter.d/sogo-auth.conf
@@ -8,7 +8,10 @@ failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' migh
 
 ignoreregex = "^<ADDR>"
 
-datepattern = {^LN-BEG}
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # 
 # DEV Notes:
diff --git a/config/filter.d/squid.conf b/config/filter.d/squid.conf
index e26cab9c..58694c48 100644
--- a/config/filter.d/squid.conf
+++ b/config/filter.d/squid.conf
@@ -9,5 +9,8 @@ failregex = ^\s+\d\s<HOST>\s+[A-Z_]+_DENIED/403 .*$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Daniel Black
 
diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf
index 0fa6eccd..a80d89e8 100644
--- a/config/filter.d/tine20.conf
+++ b/config/filter.d/tine20.conf
@@ -10,6 +10,9 @@ failregex =  ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0
 
 ignoreregex = 
 
+datepattern = ^[^-]+ -- [^-]+ -- - ({DATE})
+              {^LN-BEG}
+
 # Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766
 # Editor: Daniel Black
 # Advisor: Lars Kneschke
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 9d01a693..3849c027 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -112,6 +112,7 @@ class JailReader(ConfigReader):
 				["string", "ignorecommand", None],
 				["string", "ignoreip", None],
 				["string", "filter", ""],
+				["string", "datepattern", None],
 				["string", "action", ""]]
 
 		# Before interpolation (substitution) add static options always available as default:
@@ -195,6 +196,8 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
+		if self.__filter:
+			stream.extend(self.__filter.convert())
 		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					not self.__opts.get('backend', None).startswith("systemd"):
@@ -216,17 +219,9 @@ class JailReader(ConfigReader):
 				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
 				backend = value
-			elif opt == "maxretry":
-				stream.append(["set", self.__name, "maxretry", value])
 			elif opt == "ignoreip":
 				for ip in splitwords(value):
 					stream.append(["set", self.__name, "addignoreip", ip])
-			elif opt == "findtime":
-				stream.append(["set", self.__name, "findtime", value])
-			elif opt == "bantime":
-				stream.append(["set", self.__name, "bantime", value])
-			elif opt == "usedns":
-				stream.append(["set", self.__name, "usedns", value])
 			elif opt in ("failregex", "ignoreregex"):
 				multi = []
 				for regex in value.split('\n'):
@@ -237,10 +232,8 @@ class JailReader(ConfigReader):
 					stream.append(["multi-set", self.__name, "add" + opt, multi])
 				elif len(multi):
 					stream.append(["set", self.__name, "add" + opt, multi[0]])
-			elif opt == "ignorecommand":
-				stream.append(["set", self.__name, "ignorecommand", value])
-		if self.__filter:
-			stream.extend(self.__filter.convert())
+			elif opt not in ('action', 'filter', 'enabled'):
+				stream.append(["set", self.__name, opt, value])
 		for action in self.__actions:
 			if isinstance(action, (ConfigReaderUnshared, ConfigReader)):
 				stream.extend(action.convert())
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 7cbc0f5c..cced6d91 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -27,6 +27,7 @@ import time
 from threading import Lock
 
 from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -90,7 +91,7 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
@@ -112,8 +113,6 @@ class DateDetectorCache(object):
 		# subseconds explicit to avoid possible %m<->%d confusion
 		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
 		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
-		# TAI64N
-		self._cacheTemplate(DateTai64n())
 		# Epoch
 		self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
 		self._cacheTemplate(DateEpoch())
@@ -132,6 +131,10 @@ class DateDetectorCache(object):
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
 		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		# +00:00 Jan 23 21:59:59.011 2005 
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		# TAI64N
+		self._cacheTemplate(DateTai64n())
 		#
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
@@ -168,6 +171,7 @@ class DateDetector(object):
 	templates
 	"""
 	_defCache = DateDetectorCache()
+	_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
 
 	def __init__(self):
 		self.__templates = list()
@@ -183,9 +187,10 @@ class DateDetector(object):
 		# pre-match pattern:
 		self.__preMatch = None
 
-	def _appendTemplate(self, template):
+	def _appendTemplate(self, template, ignoreDup=False):
 		name = template.name
 		if name in self.__known_names:
+			if ignoreDup: return
 			raise ValueError(
 				"There is already a template with name %s" % name)
 		self.__known_names.add(name)
@@ -207,24 +212,56 @@ class DateDetector(object):
 			If a template already exists with the same name.
 		"""
 		if isinstance(template, str):
-			template = DatePatternRegex(template)
+			key = pattern = template
+			if '%' not in pattern:
+				key = pattern.upper()
+			template = DateDetector._patternCache.get(key)
+
+			if not template:
+				if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
+					template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
+				elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
+					template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
+				elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
+					flt = \
+						lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
+						lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
+						None
+					self.addDefaultTemplate(flt)
+					return
+				elif "{DATE}" in key:
+					self.addDefaultTemplate(
+						lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
+					return
+				else:
+					template = DatePatternRegex(pattern)
+
+			DateDetector._patternCache.set(key, template)
+
 		self._appendTemplate(template)
+		logSys.info("  date pattern `%r`: `%s`",
+			getattr(template, 'pattern', ''), template.name)
+		logSys.debug("  date pattern regex for %r: %s",
+			getattr(template, 'pattern', ''), template.regex)
 
 	def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
 		"""Add Fail2Ban's default set of date templates.
 		"""
-		for template in sorted(DateDetector._defCache.templates,
-			lambda a,b: b.hits - a.hits
-		):
+		ignoreDup = len(self.__templates) > 0
+		for template in DateDetector._defCache.templates:
 			# filter if specified:
 			if filterTemplate is not None and not filterTemplate(template): continue
 			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
 			if preMatch is not None:
-				regex = getattr(template, 'pattern', template.regex)
-				template = copy.copy(template)
-				template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
-			# append date detector template:
-			self._appendTemplate(template)
+				deftemplate = template
+				template = DateDetector._patternCache.get((preMatch, deftemplate.name))
+				if not template:
+					regex = getattr(deftemplate, 'pattern', deftemplate.regex)
+					template = copy.copy(deftemplate)
+					template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
+				DateDetector._patternCache.set((preMatch, deftemplate.name), template)
+			# append date detector template (ignore duplicate if some was added before default):
+			self._appendTemplate(template, ignoreDup=ignoreDup)
 
 	@property
 	def templates(self):
@@ -250,17 +287,22 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
-		#logSys.log(logLevel, "try to match time for line: %.120s", line)
+		# if no templates specified - default templates should be used:
+		if not len(self.__templates):
+			self.addDefaultTemplate()
+		logSys.log(logLevel-1, "try to match time for line: %.120s", line)
 		match = None
 		# first try to use last template with same start/end position:
+		ignoreBySearch = 0x7fffffff
 		i = self.__lastTemplIdx
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
-			if template.flags & DateTemplate.LINE_BEGIN:
+			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 				if logSys.getEffectiveLevel() <= logLevel-1:
 					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
 					match = template.matchDate(line)
+					ignoreBySearch = i
 			else:
 				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
 				if logSys.getEffectiveLevel() <= logLevel-1:
@@ -278,18 +320,28 @@ class DateDetector(object):
 				distance = match.start()
 				endpos = match.end()
 				# if different position, possible collision/pattern switch:
-				if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
+				if (
+					template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or 
+					(distance == self.__lastPos[0] and endpos == self.__lastEndPos[0])
+				):
 					logSys.log(logLevel, "  matched last time template #%02i", i)
 				else:
 					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
 					match = None
+			else:
+				logSys.log(logLevel, "  ** last pattern not found - pattern change, search ...")
 		# search template and better match:
 		if not match:
 			self.__lastTemplIdx = 0x7fffffff
-			logSys.log(logLevel, " search template ...")
+			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
 			found = None, 0x7fffffff, -1
 			i = 0
 			for ddtempl in self.__templates:
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
+				if i == ignoreBySearch:
+					i += 1
+					continue
 				template = ddtempl.template
 				match = template.matchDate(line)
 				if match:
@@ -298,15 +350,18 @@ class DateDetector(object):
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
 							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
+					## last (or single) template - fast stop:
+					if i+1 >= len(self.__templates):
+						break
 					## if line-begin/end anchored - stop searching:
 					if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 						break
+					## stop searching if next template still unused, but we had already hits:
+					if (distance == 0 and ddtempl.hits) and not self.__templates[i+1].template.hits:
+						break
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
-					if (
-						(distance > ddtempl.distance or distance > self.__lastPos[0]) and
-						len(self.__templates) > 1
-					):
+					if distance > ddtempl.distance or distance > self.__lastPos[0]:
 						logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
 						if distance < found[1]:
@@ -374,7 +429,7 @@ class DateDetector(object):
 				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "  got time %f for %r using template %s",
-							date[0], date[1].group(), template.name)
+							date[0], date[1].group(1), template.name)
 					return date
 			except ValueError:
 				pass
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 70a4b1b5..3a9612aa 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import re
+import re, time
 from abc import abstractmethod
 
 from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE
@@ -32,9 +32,14 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
-RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)')
-RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
-RE_DEL_WRD_BOUNDS = ( re.compile(r'^(?:\(\?\w+\))?\*\*|(?<!\\)\*\*()$'), 
+# check already grouped contains "(", but ignores char "\(" and conditional "(?(id)...)":
+RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
+RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
+
+RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*\)*)$')
+RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), 
 	                    lambda m: m.group().replace('**', '') )
 
 RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
@@ -42,8 +47,6 @@ RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
 RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
 
-_Templ_RECache = {}
-
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -66,6 +69,7 @@ class DateTemplate(object):
 		self.weight = 1.0
 		self.flags = 0
 		self.hits = 0
+		self.time = 0
 		self._regex = ""
 		self._cRegex = None
 
@@ -95,15 +99,28 @@ class DateTemplate(object):
 		re.error
 			If regular expression fails to compile
 		"""
+		# Warning: don't use lookahead for line-begin boundary, 
+		# (e. g. r"^(?:\W{0,2})?" is much faster as r"(?:^|(?<=^\W)|(?<=^\W{2}))")
+		# because it may be very slow in negative case (by long log-lines not matching pattern)
+
 		regex = regex.strip()
+		boundBegin = wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex)
+		boundEnd = wordEnd and not RE_NO_WRD_BOUND_END.search(regex)
+		# if no group add it now, should always have a group(1):
+		if not RE_GROUPED.search(regex):
+			regex = RE_GROUP[0].sub(RE_GROUP[1], regex)
 		self.flags = 0
 		# if word or line start boundary:
-		if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
+		if boundBegin:
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
-			regex = (r'(?=^|\b|\W)' if wordBegin != 'start' else r"(?:^|(?<=^\W)|(?<=^\W{2}))") + regex
-			self.name = ('{*WD-BEG}' if wordBegin != 'start' else '{^LN-BEG}') + self.name
+			if wordBegin != 'start':
+				regex = r'(?:^|\b|\W)' + regex
+				self.name = '{*WD-BEG}' + self.name
+			else:
+				regex = r"^(?:\W{0,2})?" + regex
+				self.name = '{^LN-BEG}' + self.name
 		# if word end boundary:
-		if wordEnd and not RE_NO_WRD_BOUND_END.search(regex):
+		if boundEnd:
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
 			self.name += '{*WD-END}'
@@ -112,6 +129,7 @@ class DateTemplate(object):
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
+		logSys.debug('  constructed regex %s', regex)
 		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
@@ -123,6 +141,7 @@ class DateTemplate(object):
 		"""
 		if not self._cRegex:
 			try:
+				# print('*'*10 + (' compile - %-30.30s -- %s' % (getattr(self, 'pattern', self.regex), self.name)))
 				self._cRegex = re.compile(self.regex)
 			except Exception as e:
 				logSys.error('Compile %r failed, expression %r', self.name, self.regex)
@@ -136,6 +155,7 @@ class DateTemplate(object):
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		if dateMatch:
 			self.hits += 1
+		# print('*'*10 + ('[%s] - %-30.30s -- %s' % ('*' if dateMatch else ' ', getattr(self, 'pattern', self.regex), self.name)))
 		return dateMatch
 
 	@abstractmethod
@@ -175,10 +195,10 @@ class DateEpoch(DateTemplate):
 		DateTemplate.__init__(self)
 		self.name = "Epoch"
 		if not lineBeginOnly:
-			regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
+			regex = r"((?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
 			self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
 		else:
-			regex = r"(?P<square>(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))"
+			regex = r"((?P<square>(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))"
 			self.setRegex(regex, wordBegin='start', wordEnd=True)
 
 	def getDate(self, line, dateMatch=None):
@@ -199,7 +219,7 @@ class DateEpoch(DateTemplate):
 			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
-			return (float(dateMatch.group()), dateMatch)
+			return (float(dateMatch.group(1)), dateMatch)
 		return None
 
 
@@ -244,7 +264,13 @@ class DatePatternRegex(DateTemplate):
 		self.setRegex(pattern)
 
 	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
+		# original pattern:
 		self._pattern = pattern
+		# if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin:
+		if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern):
+			pattern = RE_EXLINE_BOUND_BEG.sub('', pattern)
+			wordBegin = 'start'
+		# wrap to regex:
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
 		self.name = fmt % self._patternName
 		regex = fmt % timeRE
@@ -285,12 +311,11 @@ class DateTai64n(DateTemplate):
 	regex
 	"""
 
-	def __init__(self):
+	def __init__(self, wordBegin=False):
 		DateTemplate.__init__(self)
 		self.name = "TAI64N"
 		# We already know the format for TAI64N
-		# yoh: we should not add an additional front anchor
-		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
+		self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
@@ -310,7 +335,7 @@ class DateTai64n(DateTemplate):
 			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
-			value = dateMatch.group()
+			value = dateMatch.group(1)
 			seconds_since_epoch = value[2:17]
 			# convert seconds from HEX into local time stamp
 			return (int(seconds_since_epoch, 16), dateMatch)
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index cbf59fcb..7f7584cf 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -35,7 +35,6 @@ from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
-from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
@@ -94,7 +93,6 @@ class Filter(JailThread):
 		self.ticks = 0
 
 		self.dateDetector = DateDetector()
-		self.dateDetector.addDefaultTemplate()
 		logSys.debug("Created %s" % self)
 
 	def __repr__(self):
@@ -258,34 +256,12 @@ class Filter(JailThread):
 			self.dateDetector = None
 			return
 		else:
-			key = pattern.upper()
-			if key == "EPOCH":
-				template = DateEpoch()
-				template.name = "Epoch"
-			elif key == "TAI64N":
-				template = DateTai64n()
-				template.name = "TAI64N"
-			elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
-				self.dateDetector = DateDetector()
-				flt = \
-					lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
-					lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
-					None
-				self.dateDetector.addDefaultTemplate(flt)
-				return
-			elif "{DATE}" in key:
-				self.dateDetector = DateDetector()
-				self.dateDetector.addDefaultTemplate(
-					lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
-				return
-			else:
-				template = DatePatternRegex(pattern)
-		self.dateDetector = DateDetector()
-		self.dateDetector.appendTemplate(template)
-		logSys.info("  date pattern `%r`: `%s`",
-			pattern, template.name)
-		logSys.debug("  date pattern regex for %r: %s",
-			pattern, template.regex)
+			dd = DateDetector()
+			if not isinstance(pattern, (list, tuple)):
+				pattern = filter(bool, map(str.strip, re.split('\n+', pattern)))
+			for pattern in pattern:
+				dd.appendTemplate(pattern)
+			self.dateDetector = dd
 
 	##
 	# Get the date detector pattern, or Default Detectors if not changed
@@ -295,7 +271,8 @@ class Filter(JailThread):
 	def getDatePattern(self):
 		if self.dateDetector is not None:
 			templates = self.dateDetector.templates
-			if len(templates) > 2:
+			# lazy template init, by first match
+			if not len(templates) or len(templates) > 2:
 				return None, "Default Detectors"
 			elif len(templates):
 				if hasattr(templates[0], "pattern"):
@@ -303,6 +280,7 @@ class Filter(JailThread):
 				else:
 					pattern = None
 				return pattern, templates[0].name
+		return None
 
 	##
 	# Set the maximum retry value.
@@ -483,9 +461,9 @@ class Filter(JailThread):
 			(timeMatch, template) = self.dateDetector.matchTime(l)
 			if timeMatch:
 				tupleLine  = (
-					l[:timeMatch.start()],
-					l[timeMatch.start():timeMatch.end()],
-					l[timeMatch.end():],
+					l[:timeMatch.start(1)],
+					l[timeMatch.start(1):timeMatch.end(1)],
+					l[timeMatch.end(1):],
 					(timeMatch, template)
 				)
 			else:
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index da00a57e..643984d3 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -55,8 +55,6 @@ timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
 # respect possible run in the test-cases (alternate date used there):
 timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3)
 timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3)
-# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates:
-timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))"
 
 def getTimePatternRE():
 	keys = timeRE.keys()
@@ -70,7 +68,6 @@ def getTimePatternRE():
 		'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
 		'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%",
 		'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name",
-		'ExLB': '{^LN-BEG}',
 	}
 	for key in set(keys) - set(names): # may not have them all...
 		if key.startswith('Ex'):
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 2f5be043..a0dfc639 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -303,7 +303,7 @@ class Transmitter:
 					actionvalue = command[4]
 					setattr(action, actionkey, actionvalue)
 					return getattr(action, actionkey)
-		raise Exception("Invalid command (no set action or not yet implemented)")
+		raise Exception("Invalid command %r (no set action or not yet implemented)" % (command[1],))
 	
 	def __commandGet(self, command):
 		name = command[0]
diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 64db44e7..2f3b6723 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -32,6 +32,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 		
 		def setUp(self):
 			"""Call before every test case."""
+			super(BadIPsActionTest, self).setUp()
 			unittest.F2B.SkipIfNoNetwork()
 
 			self.jail = DummyJail()
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 5c8b1923..dc28d5c0 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -45,6 +45,7 @@ class SMTPActionTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(SMTPActionTest, self).setUp()
 		self.jail = DummyJail()
 		pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")
 		pythonModuleName = os.path.basename(pythonModule.rstrip(".py"))
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 4d964425..2c9d6601 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -32,6 +32,7 @@ from ..server.ticket import BanTicket
 class AddFailure(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
+		super(AddFailure, self).setUp()
 		self.__ticket = BanTicket('193.168.0.128', 1167605999.0)
 		self.__banManager = BanManager()
 
@@ -134,6 +135,7 @@ class AddFailure(unittest.TestCase):
 class StatusExtendedCymruInfo(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
+		super(StatusExtendedCymruInfo, self).setUp()
 		unittest.F2B.SkipIfNoNetwork()
 		self.__ban_ip = "93.184.216.34"
 		self.__asn = "15133"
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 0390a8ff..7d20e84e 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -32,6 +32,7 @@ class BeautifierTest(unittest.TestCase):
 
 	def setUp(self):
 		""" Call before every test case """
+		super(BeautifierTest, self).setUp()
 		self.b = Beautifier()
 
 	def tearDown(self):
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index e68523c2..36722ff1 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -55,6 +55,7 @@ class ConfigReaderTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(ConfigReaderTest, self).setUp()
 		self.d = tempfile.mkdtemp(prefix="f2b-temp")
 		self.c = ConfigReaderUnshared(basedir=self.d)
 
diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
index df30d725..8d1f2ae9 100644
--- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf
+++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf
@@ -20,3 +20,8 @@ failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
 # just to test multiple ignoreregex:
 ignoreregex = ^%(__prefix_line)sF2B: error from 192.0.2.251$
               ^%(__prefix_line)sF2B: error from 192.0.2.252$
+
+# specify only exact date patterns, +1 with %%Y to test usage of last known date by wrong dates like 0000-00-00...
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)?
+              {^LN-BEG}%%Y(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 6bf92582..fdb7ec71 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -58,6 +58,8 @@ class DateDetectorTest(LogCaptureTestCase):
 		return self.__datedetector
 	
 	def testGetEpochTime(self):
+		self.__datedetector = DateDetector()
+		self.__datedetector.appendTemplate('EPOCH')
 		# correct epoch time, using all variants:
 		for dateUnix in (1138049999, 32535244799):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
@@ -67,7 +69,7 @@ class DateDetectorTest(LogCaptureTestCase):
 				self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,))
 				( datelog, matchlog ) = datelog
 				self.assertEqual(int(datelog), dateUnix)
-				self.assertIn(matchlog.group(), (str(dateUnix), str(dateUnix)+'.555'))
+				self.assertIn(matchlog.group(1), (str(dateUnix), str(dateUnix)+'.555'))
 		# wrong, no epoch time (< 10 digits, more as 11 digits, begin/end of word) :
 		for dateUnix in ('123456789', '9999999999999999', '1138049999A', 'A1138049999'):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
@@ -85,7 +87,7 @@ class DateDetectorTest(LogCaptureTestCase):
 		#      of fail2ban -- we just ignore incorrect day of the week
 		( datelog, matchlog ) = self.datedetector.getTime(log)
 		self.assertEqual(datelog, dateUnix)
-		self.assertEqual(matchlog.group(), 'Jan 23 21:59:59')
+		self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
 
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
@@ -150,7 +152,7 @@ class DateDetectorTest(LogCaptureTestCase):
 					( logUnix, logMatch ) = logtime
 					self.assertEqual(logUnix, dateUnix,
 						"getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
-					self.assertEqual(logMatch.group(), rdate)
+					self.assertEqual(logMatch.group(1), rdate)
 				else:
 					self.assertEqual(logtime, None,
 						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
@@ -164,7 +166,7 @@ class DateDetectorTest(LogCaptureTestCase):
 					( logUnix, logMatch ) = logtime
 					self.assertEqual(logUnix, dateUnix,
 						"getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
-					self.assertEqual(logMatch.group(), rdate)
+					self.assertEqual(logMatch.group(1), rdate)
 				else:
 					self.assertEqual(logtime, None,
 						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
@@ -183,23 +185,23 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertNotEqual(logdate, None)
 		( logTime, logMatch ) = logdate
 		self.assertEqual(logTime, mu)
-		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
+		self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17')
 		# confuse it with year being at the end
 		for i in xrange(10):
 			( logTime, logMatch ) =	self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
 			self.assertEqual(logTime, mu)
-			self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
+			self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17')
 		# and now back to the original
 		( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
 		self.assertEqual(logTime, mu)
-		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
+		self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17')
 
 	def testDateTemplate(self):
 		t = DateTemplate()
 		t.setRegex('^a{3,5}b?c*$')
-		self.assertEqual(t.regex, '^a{3,5}b?c*$')
+		self.assertEqual(t.regex, '^(a{3,5}b?c*)$')
 		self.assertRaises(Exception, t.getDate, '')
-		self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+		self.assertEqual(t.matchDate('aaaac').group(1), 'aaaac')
 
 		## no word boundaries left and right:
 		t = DatePatternRegex()
@@ -208,22 +210,22 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertFalse('**' in t.regex)
 		# match date:
 		dt = 'TIME:20050102T010203'
-		self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt)
-		self.assertEqual(t.matchDate(dt).group(), dt)
+		self.assertEqual(t.matchDate('X' + dt + 'X').group(1), dt)
+		self.assertEqual(t.matchDate(dt).group(1), dt)
 		# wrong year (for exact %ExY):
 		dt = 'TIME:50050102T010203'
 		self.assertFalse(t.matchDate(dt))
 
-		## start boundary left and word boundary right:
+		## start boundary left and word boundary right (automatically if not **):
 		t = DatePatternRegex()
-		t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS'
+		t.pattern = '{^LN-BEG}time:%ExY%Exm%ExdT%ExH%ExM%ExS'
 		self.assertTrue('^' in t.regex)
 		# try match date:
 		dt = 'time:20050102T010203'
 		self.assertFalse(t.matchDate('X' + dt))
 		self.assertFalse(t.matchDate(dt + 'X'))
-		self.assertEqual(t.matchDate('##' + dt + '...').group(), dt)
-		self.assertEqual(t.matchDate(dt).group(), dt)
+		self.assertEqual(t.matchDate('##' + dt + '...').group(1), dt)
+		self.assertEqual(t.matchDate(dt).group(1), dt)
 		# case sensitive:
 		dt = 'TIME:20050102T010203'
 		self.assertFalse(t.matchDate(dt))
@@ -232,9 +234,9 @@ class DateDetectorTest(LogCaptureTestCase):
 		t = DatePatternRegex()
 		t.pattern = '^%Y %b %d'
 		self.assertTrue('(?iu)' in t.regex)
-		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
-		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
-		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
+		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
+		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
 
 	def testAmbiguousInOrderedTemplates(self):
 		dd = self.datedetector
@@ -259,7 +261,7 @@ class DateDetectorTest(LogCaptureTestCase):
 				logSys.debug('Line: %s', line)
 				match, template = dd.matchTime(line)
 				self.assertTrue(match)
-				self.assertEqual(match.group(), debit)
+				self.assertEqual(match.group(1), debit)
 
 	def testLowLevelLogging(self):
 		# test coverage for the deep (heavy) debug messages:
@@ -363,31 +365,30 @@ class CustomDateFormatsTest(unittest.TestCase):
 			('200333 010203',   r'%Y%m%d %H%M%S',             "text:200333 010203 | date:20031230 010203"),
 			('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
 			('20031230 010203', None,                         "text:200333 010203 | date:20031230 010203"),
-			# Explicit bound in start of the line using %ExLB key,
+			# Explicit bound in start of the line using {^LN-BEG} key,
 			# (negative) in the 1st case without line begin boundary - wrong date may be found,
 			# (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found)
 			# (positive) and in 3th case with line begin boundary - find the correct date
 			("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS",      "00001230 010203 - 20030101 000000"),
-			(None,              "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
-			("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
-			# Explicit bound in start of the line using %ExLB key, 
+			(None,              "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
+			("20031230 010203", "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
+			# Explicit bound in start of the line using {^LN-BEG} key, 
 			# up to 2 non-alphanumeric chars front, ** - no word boundary on the right
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
 		):
 			logSys.debug('== test: %r', (matched, dp, line))
 			if dp is None:
 				dd = defDD
 			else:
-				dp = DatePatternRegex(dp)
 				dd = DateDetector()
 				dd.appendTemplate(dp)
 			date = dd.getTime(line)
 			if matched:
 				self.assertTrue(date)
-				self.assertEqual(matched, date[1].group())
+				self.assertEqual(matched, date[1].group(1))
 			else:
 				self.assertEqual(date, None)
 
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e212ff23..8e25fd77 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -743,6 +743,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"maxretry = 3",
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",
+				"datepattern = {^LN-BEG}EPOCH",
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index c62377a9..5767a3f7 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -101,6 +101,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWrongIngnoreRE(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "{^LN-BEG}EPOCH",
 			"test", r".*? from <HOST>$", r".**"
 		)
 		self.assertFalse(fail2banRegex.start(opts, args))
@@ -108,6 +109,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectFound(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--print-no-missed",
 			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
 			r"Authentication failure for .*? from <HOST>$"
@@ -136,6 +138,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -151,6 +154,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1raw(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--raw",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -160,6 +164,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1raw_noDns(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--raw", "--usedns=no",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -169,6 +174,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_2(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
@@ -178,6 +184,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testVerbose(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--verbose", "--verbose-date", "--print-no-missed",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
@@ -190,6 +197,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWronChar(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
 		self.assertTrue(fail2banRegex.start(opts, args))
@@ -203,6 +211,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWronCharDebuggex(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--debuggex", "--print-all-matched",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index 6e7bf367..ed68e2a8 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -36,6 +36,7 @@ class AddFailure(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(AddFailure, self).setUp()
 		self.__items = None
 		self.__failManager = FailManager()
 
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index 51d3974c..2044c387 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -30,8 +30,8 @@ Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-
 # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
 [Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
 
-# -- wrong time direct in journal-line (using precise year pattern):
-# failJSON: { "match": false}
+# -- wrong time direct in journal-line (used last known date):
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.1
 # -- wrong time after newline in message (plist without escaped newlines):
 # failJSON: { "match": false }
@@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
              applicationDate = "0000-12-30 00:00:00 +0000";
 # failJSON: { "match": false }
 }
-# -- wrong time direct in journal-line (using precise year pattern):
-# failJSON: { "match": false}
+# -- wrong time direct in journal-line (used last known date):
+# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2
 
 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" }
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ea996072..227050f2 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -270,6 +270,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p
 class BasicFilter(unittest.TestCase):
 
 	def setUp(self):
+		super(BasicFilter, self).setUp()
 		self.filter = Filter('name')
 
 	def testGetSetUseDNS(self):
@@ -363,6 +364,7 @@ class IgnoreIP(LogCaptureTestCase):
 		setUpMyTime()
 		self.filter.addIgnoreIP('192.168.1.0/25')
 		self.filter.addFailRegex('<HOST>')
+		self.filter.setDatePattern('{^LN-BEG}EPOCH')
 		self.filter.processLineAndAdd('1387203300.222 192.168.1.32')
 		self.assertLogged('Ignore 192.168.1.32')
 		tearDownMyTime()
@@ -461,6 +463,7 @@ class LogFileFilterPoll(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(LogFileFilterPoll, self).setUp()
 		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(LogFileFilterPoll.FILENAME)
 
@@ -653,6 +656,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		self.assertLogged('Unable to open %s' % self.name)
 
 	def testErrorProcessLine(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 		self.filter.sleeptime /= 1000.0
 		## produce error with not callable processLine:
 		_org_processLine = self.filter.processLine
@@ -715,6 +720,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		pass
 
 	def testNewChangeViaGetFailures_simple(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		# suck in lines from this sample log file
 		self.filter.getFailures(self.name)
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
@@ -730,6 +737,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01)
 
 	def testNewChangeViaGetFailures_rewrite(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		#
 		# if we rewrite the file at once
 		self.file.close()
@@ -748,6 +757,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01)
 
 	def testNewChangeViaGetFailures_move(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		#
 		# if we move file into a new location while it has been open already
 		self.file.close()
@@ -769,6 +780,7 @@ class CommonMonitorTestCase(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(CommonMonitorTestCase, self).setUp()
 		self._failTotal = 0
 
 	def waitFailTotal(self, count, delay=1.):
@@ -819,6 +831,8 @@ def get_monitor_failures_testcase(Filter_):
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
 			self.filter.addLogPath(self.name, autoSeek=False)
+			# speedup search using exact date pattern:
+			self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 			self.filter.active = True
 			self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
 			self.filter.start()
@@ -1223,6 +1237,8 @@ class GetFailures(LogCaptureTestCase):
 		self.jail = DummyJail()
 		self.filter = FileFilter(self.jail)
 		self.filter.active = True
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		# TODO Test this
 		#self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 		#self.filter.setTimePattern("%b %d %H:%M:%S")
@@ -1329,6 +1345,11 @@ class GetFailures(LogCaptureTestCase):
 		output = (('212.41.96.186', 4, 1124013600.0),
 				  ('212.41.96.185', 2, 1124013598.0))
 
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?',
+			'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?',
+			'^EPOCH'
+		))
 		self.filter.setMaxRetry(2)
 		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
@@ -1358,6 +1379,8 @@ class GetFailures(LogCaptureTestCase):
 				if enc is not None:
 					self.tearDown();self.setUp();
 					self.filter.setLogEncoding(enc);
+				# speedup search using exact date pattern:
+				self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 				self.assertNotLogged('Error decoding line');
 				self.filter.addLogPath(fname)
 				self.filter.addFailRegex(failregex)
@@ -1533,6 +1556,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(DNSUtilsNetworkTests, self).setUp()
 		unittest.F2B.SkipIfNoNetwork()
 
 	def test_IPAddr(self):
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 908e4f6c..cd841a0f 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -86,6 +86,7 @@ def _getSysPythonVersion():
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
+		super(SetupTest, self).setUp()
 		unittest.F2B.SkipIfFast()
 		setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py')
 		self.setup = os.path.exists(setup) and setup or None
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 31b1812e..0ed3e764 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -43,6 +43,7 @@ class FilterSamplesRegex(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(FilterSamplesRegex, self).setUp()
 		self.filter = Filter(None)
 		self.filter.active = True
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index cd7ce5c9..03fd1a58 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -65,7 +65,7 @@ class TransmitterBase(unittest.TestCase):
 	
 	def setUp(self):
 		"""Call before every test case."""
-		#super(TransmitterBase, self).setUp()
+		super(TransmitterBase, self).setUp()
 		self.transm = self.server._Server__transm
 		# To test thransmitter we don't need to start server...
 		#self.server.start('/dev/null', '/dev/null', force=False)
@@ -301,9 +301,11 @@ class Transmitter(TransmitterBase):
 			("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"),
 			jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "Epoch", (None, "Epoch"), jail=self.jailName)
+			"datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
+			"datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName)
+		self.setGetTest(
+			"datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName)
 		self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
 
 	def testJailUseDNS(self):
diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py
index 5bf0be57..1a94a952 100644
--- a/fail2ban/tests/sockettestcase.py
+++ b/fail2ban/tests/sockettestcase.py
@@ -41,6 +41,7 @@ class Socket(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(Socket, self).setUp()
 		self.server = AsyncServer(self)
 		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'socket')
 		os.close(sock_fd)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 22079456..fc60f87b 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -526,6 +526,16 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 			self.fail(msg)
 	unittest.TestCase.assertNotIn = assertNotIn
 
+_org_setUp = unittest.TestCase.setUp
+def _customSetUp(self):
+	# print('=='*10, self)
+	if unittest.F2B.log_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
+		print("")
+		logSys.debug('='*10 + ' %s ' + '='*20, self.id())
+	_org_setUp(self)
+
+unittest.TestCase.setUp = _customSetUp
+
 
 class LogCaptureTestCase(unittest.TestCase):
 
@@ -601,12 +611,11 @@ class LogCaptureTestCase(unittest.TestCase):
 		# Let's log everything into a string
 		self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy)
 		logSys.handlers = [self._log]
-		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
-			print("")
+		if self._old_level <= logging.DEBUG:
 			logSys.handlers += self._old_handlers
-			logSys.debug('='*10 + ' %s ' + '='*20, self.id())
-		else:
+		else: # lowest log level to capture messages
 			logSys.setLevel(logging.DEBUG)
+		super(LogCaptureTestCase, self).setUp()
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -615,6 +624,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		logSys = getLogger("fail2ban")
 		logSys.handlers = self._old_handlers
 		logSys.level = self._old_level
+		super(LogCaptureTestCase, self).tearDown()
 
 	def _is_logged(self, *s, **kwargs):
 		logged = self._log.getvalue()

From faee5f1fdc4a91dd39dff230ee0aea2d0fefcf70 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 7 Oct 2016 18:33:01 +0200
Subject: [PATCH 231/623] better caching (thereby better performance), better
 recognition of similar regex

---
 config/filter.d/ejabberd-auth.conf |   5 +-
 fail2ban/server/datedetector.py    | 101 ++++++++++++++++++-----------
 fail2ban/server/datetemplate.py    |   5 +-
 fail2ban/tests/servertestcase.py   |   8 +--
 4 files changed, 73 insertions(+), 46 deletions(-)

diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf
index edd87463..56517489 100644
--- a/config/filter.d/ejabberd-auth.conf
+++ b/config/filter.d/ejabberd-auth.conf
@@ -34,4 +34,7 @@ maxlines = 2
 #
 journalmatch = 
 
-datepattern = ^(?:=[^=]+={3,} )?({DATE})
+#datepattern = ^(?:=[^=]+={3,} )?({DATE})
+# explicit time format using prefix =...==== and no date in second string begins with I(...)... 
+datepattern = ^(?:=[^=]+={3,} )?(%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?)
+              ^I\(()**
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index cced6d91..458f8d11 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -36,6 +36,46 @@ logSys = getLogger(__name__)
 logLevel = 6
 
 RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE)
+DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
+
+
+def _getPatternTemplate(pattern, key=None):
+	if key is None:
+		key = pattern
+		if '%' not in pattern:
+			key = pattern.upper()
+	template = DD_patternCache.get(key)
+
+	if not template:
+		if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
+			template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
+		elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
+			template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
+		else:
+			template = DatePatternRegex(pattern)
+
+	DD_patternCache.set(key, template)
+	return template
+
+def _getAnchoredTemplate(template, wrap=lambda s: '{^LN-BEG}' + s):
+	# wrap name:
+	name = wrap(template.name)
+	# try to find in cache (by name):
+	template2 = DD_patternCache.get(name)
+	if not template2:
+		# wrap pattern (or regexp if not pattern template):
+		regex = wrap(getattr(template, 'pattern', template.regex))
+		if hasattr(template, 'pattern'):
+			# try to find in cache (by pattern):
+			template2 = DD_patternCache.get(regex)
+		# make duplicate and set new anchored regex:
+		if not template2:
+			if not hasattr(template, 'pattern'):
+				template2 = _getPatternTemplate(name)
+			else:
+				template2 = _getPatternTemplate(regex)
+	return template2
+
 
 
 class DateDetectorCache(object):
@@ -57,22 +97,20 @@ class DateDetectorCache(object):
 			self._addDefaultTemplate()
 			return self.__templates
 
-	def _cacheTemplate(self, template, lineBeginOnly=False):
+	def _cacheTemplate(self, template):
 		"""Cache Fail2Ban's default template.
 
 		"""
 		if isinstance(template, str):
-			# exact given template with word benin-end boundary:
-			if not lineBeginOnly:
-				template = DatePatternRegex(template)
-			else:
-				template = DatePatternRegex(template, wordBegin='start')
-		# additional template, that prefers datetime at start of a line (safety+performance feature):
-		if not lineBeginOnly and hasattr(template, 'regex'):
-			template2 = copy.copy(template)
-			regex = getattr(template, 'pattern', template.regex)
-			template2.setRegex(regex, wordBegin='start', wordEnd=True)
-			if template2.name != template.name:
+			# exact given template with word begin-end boundary:
+			template = _getPatternTemplate(template)
+		# if not already line-begin anchored, additional template, that prefers datetime 
+		# at start of a line (safety+performance feature):
+		name = template.name
+		if not name.startswith('{^LN-BEG}') and not name.startswith('^') and hasattr(template, 'regex'):
+			template2 = _getAnchoredTemplate(template)
+			# prevent to add duplicates:
+			if template2.name != name:
 				# increase weight of such templates, because they should be always
 				# preferred in template sorting process (bubble up):
 				template2.weight = 100.0
@@ -114,18 +152,17 @@ class DateDetectorCache(object):
 		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
 		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
 		# Epoch
-		self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
-		self._cacheTemplate(DateEpoch())
+		self._cacheTemplate('EPOCH')
 		# Only time information in the log
-		self._cacheTemplate("%H:%M:%S", lineBeginOnly=True)
+		self._cacheTemplate("{^LN-BEG}%H:%M:%S")
 		# <09/16/08@05:03:30>
-		self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True)
+		self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>")
 		# MySQL: 130322 11:46:11
 		self._cacheTemplate("%Exy%Exm%Exd  ?%H:%M:%S")
 		# Apache Tomcat
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True)
+		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
 		# 20050123T215959, 20050123 215959
 		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# prefixed with optional named time zone (monit):
@@ -134,7 +171,7 @@ class DateDetectorCache(object):
 		# +00:00 Jan 23 21:59:59.011 2005 
 		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# TAI64N
-		self._cacheTemplate(DateTai64n())
+		self._cacheTemplate("TAI64N")
 		#
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
@@ -171,7 +208,6 @@ class DateDetector(object):
 	templates
 	"""
 	_defCache = DateDetectorCache()
-	_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
 
 	def __init__(self):
 		self.__templates = list()
@@ -215,18 +251,11 @@ class DateDetector(object):
 			key = pattern = template
 			if '%' not in pattern:
 				key = pattern.upper()
-			template = DateDetector._patternCache.get(key)
-
+			template = DD_patternCache.get(key)
 			if not template:
-				if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
-					template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
-				elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
-					template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
-				elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
+				if key in ("{^LN-BEG}", "{DEFAULT}"):
 					flt = \
-						lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
-						lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
-						None
+						lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else None
 					self.addDefaultTemplate(flt)
 					return
 				elif "{DATE}" in key:
@@ -234,9 +263,9 @@ class DateDetector(object):
 						lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
 					return
 				else:
-					template = DatePatternRegex(pattern)
+					template = _getPatternTemplate(pattern, key)
 
-			DateDetector._patternCache.set(key, template)
+			DD_patternCache.set(key, template)
 
 		self._appendTemplate(template)
 		logSys.info("  date pattern `%r`: `%s`",
@@ -253,13 +282,9 @@ class DateDetector(object):
 			if filterTemplate is not None and not filterTemplate(template): continue
 			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
 			if preMatch is not None:
-				deftemplate = template
-				template = DateDetector._patternCache.get((preMatch, deftemplate.name))
-				if not template:
-					regex = getattr(deftemplate, 'pattern', deftemplate.regex)
-					template = copy.copy(deftemplate)
-					template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
-				DateDetector._patternCache.set((preMatch, deftemplate.name), template)
+				# get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}:
+				template = _getAnchoredTemplate(template,
+					wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
 
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 3a9612aa..f670cbbd 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -115,15 +115,14 @@ class DateTemplate(object):
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
 			if wordBegin != 'start':
 				regex = r'(?:^|\b|\W)' + regex
-				self.name = '{*WD-BEG}' + self.name
 			else:
 				regex = r"^(?:\W{0,2})?" + regex
-				self.name = '{^LN-BEG}' + self.name
+				if not self.name.startswith('{^LN-BEG}'):
+					self.name = '{^LN-BEG}' + self.name
 		# if word end boundary:
 		if boundEnd:
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
-			self.name += '{*WD-END}'
 		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
 		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 03fd1a58..724e3bb9 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -298,14 +298,14 @@ class Transmitter(TransmitterBase):
 
 	def testDatePattern(self):
 		self.setGetTest("datepattern", "%%%Y%m%d%H%M%S",
-			("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"),
+			("%%%Y%m%d%H%M%S", "%YearMonthDay24hourMinuteSecond"),
 			jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName)
+			"datepattern", "Epoch", (None, "Epoch"), jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName)
+			"datepattern", "^Epoch", (None, "{^LN-BEG}Epoch"), jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName)
+			"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
 		self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
 
 	def testJailUseDNS(self):

From ffa97054120d487dd5a19b6f88d2ec565c45000b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 17 Oct 2016 12:09:23 +0200
Subject: [PATCH 232/623] fixed UTC/GMT named time zone using `%Z` and `%z`
 patterns (special case with 0 zone offset); Currently still ignores another
 named zones, because fail2ban assumes that the given date is in the current
 default zone. Closes gh-1575

---
 fail2ban/server/strptime.py                   |  8 ++++++--
 fail2ban/tests/datedetectortestcase.py        | 10 +++++++++-
 fail2ban/tests/files/logs/zzz-generic-example |  8 ++++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 643984d3..cdfe0e0e 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -41,7 +41,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
 #timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
 timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
-timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
+timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-]\d{2}(?::?[0-5]\d)?)"
 
 # Extend build-in TimeRE with some exact patterns
 # exact two-digit patterns:
@@ -183,7 +183,7 @@ def reGroupDictStrptime(found_dict, msec=False):
 				week_of_year_start = 0
 		elif key == 'z':
 			z = val
-			if z == "Z":
+			if z in ("Z", "UTC", "GMT"):
 				tzoffset = 0
 			else:
 				tzoffset = int(z[1:3]) * 60 # Hours...
@@ -191,6 +191,10 @@ def reGroupDictStrptime(found_dict, msec=False):
 					tzoffset += int(z[-2:]) # ...and minutes
 				if z.startswith("-"):
 					tzoffset = -tzoffset
+		elif key == 'Z':
+			z = val
+			if z in ("UTC", "GMT"):
+				tzoffset = 0
 
 	# Fail2Ban will assume it's this year
 	assume_year = False
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index fdb7ec71..695f099c 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -378,6 +378,11 @@ class CustomDateFormatsTest(unittest.TestCase):
 			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
 			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
 			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
+			# UTC/GMT time zone offset (with %z and %Z):
+			(1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03] server ..."),
+			(1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03] server ..."),
+			(1072746123.0,        "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 UTC] server ..."),
+			(1072746123.0,        "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 UTC] server ..."),
 		):
 			logSys.debug('== test: %r', (matched, dp, line))
 			if dp is None:
@@ -388,7 +393,10 @@ class CustomDateFormatsTest(unittest.TestCase):
 			date = dd.getTime(line)
 			if matched:
 				self.assertTrue(date)
-				self.assertEqual(matched, date[1].group(1))
+				if isinstance(matched, basestring):
+					self.assertEqual(matched, date[1].group(1))
+				else:
+					self.assertEqual(matched, date[0])
 			else:
 				self.assertEqual(date, None)
 
diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example
index 2044c387..d0c31740 100644
--- a/fail2ban/tests/files/logs/zzz-generic-example
+++ b/fail2ban/tests/files/logs/zzz-generic-example
@@ -46,6 +46,14 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
 # failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2
 
+# -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST):
+# failJSON: { "time": "2005-06-21T16:55:09", "match": true , "host": "192.0.2.09" }
+2005-06-21 16:55:09 machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.09
+# failJSON: { "time": "2005-06-21T16:55:10", "match": true , "host": "192.0.2.10" }
+2005-06-21 14:55:10 UTC machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.10
+# failJSON: { "time": "2005-06-21T16:55:11", "match": true , "host": "192.0.2.11" }
+2005-06-21 14:55:11 GMT machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.11
+
 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" }
 [Jun 21 16:56:02] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.250
 # failJSON: { "match": false, "desc": "test 1st ignoreregex" }

From c8b036456dad258a4b9d38168530933bc02cc71c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 17 Oct 2016 12:11:41 +0200
Subject: [PATCH 233/623] changelog entries

---
 ChangeLog | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 16c65556..ff728685 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -13,6 +13,15 @@ TODO: implementing of options resp. other tasks from PR #1346
 
 ### Fixes
 * [Grave] memory leak's fixed (gh-1277, gh-1234)
+* [Grave] Misleading date patterns defined more precisely (using extended syntax
+  `%Ex[mdHMS]` for exact two-digit match or e. g. `%ExY` as more precise year 
+  pattern, within same century of last year and the next 3 years)
+* [Grave] extends date detector template with distance (position of match in 
+  log-line), to prevent grave collision using (re)ordered template list (e.g.
+  find-spot of wrong date-match inside foreign input, misleading date patterns
+  by ambiguous formats, etc.)
+* Distance collision check always prefers template with shortest distance
+  (left for right) if date pattern is not anchored
 * Tricky bug fix: last position of log file will be never retrieved (gh-795),
   because of CASCADE all log entries will be deleted from logs table together with jail, 
   if used "INSERT OR REPLACE" statement
@@ -28,6 +37,11 @@ TODO: implementing of options resp. other tasks from PR #1346
 * Pyinotify-backend: stability fix for sporadically errors in multi-threaded
   environment (without lock)
 * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values
+* Fixed UTC/GMT named time zone, using `%Z` and `%z` patterns 
+  (special case with 0 zone offset, see gh-1575)
+* `filter.d/freeswitch.conf`
+    - Optional prefixes (server, daemon, dual time) if systemd daemon logs used (gh-1548)
+    - User part rewritten to accept IPv6 resp. domain after "@" (gh-1548)
 
 ### New Features
 * IPv6 support:
@@ -118,6 +132,22 @@ fail2ban-client set loglevel INFO
   - new replacement for `<ADDR>` in opposition to `<HOST>`, for separate
     usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6`
     together, without host (dns)
+* More precise date template handling (WARNING: theoretically possible incompatibilities):
+  - datedetector rewritten more strict as earlier;
+  - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`);
+  - more as one date pattern can be specified using option `datepattern` now 
+    (new-line separated);
+  - some default options like `datepattern` can be specified directly in 
+    section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]`
+    section, because of performance (each extra section costs time);
+  - option `datepattern` can be specified in jail also (e. g. jails without filters 
+    or custom log-format, new-line separated for multiple patterns);
+  - if first unnamed group specified in pattern, only this will be cut out from
+    search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match 
+    pattern, and leaves `date:[] ...` for searching in filter);
+  - faster match and fewer searching of appropriate templates
+    (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
+  - several standard filters extended with exact prefixed or anchored date templates;
 * fail2ban-testcases:
   - `assertLogged` extended with parameter wait (to wait up to specified timeout,
     before we throw assert exception) + test cases rewritten using that

From 58717c185484e643b808c3c77feb2520d36e88f3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 17 Oct 2016 13:26:24 +0200
Subject: [PATCH 234/623] fail2ban-testcases: persistently set (python) time
 zone to CET during test cases process (used in zone-related test-cases)

---
 fail2ban/tests/utils.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index fc60f87b..ae543e3d 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -260,6 +260,10 @@ def initTests(opts):
 			raise unittest.SkipTest('Skip test because of "--no-network"')
 		unittest.F2B.SkipIfNoNetwork = F2B_SkipIfNoNetwork
 
+	# persistently set time zone to CET (used in zone-related test-cases),
+	# yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match
+	os.environ['TZ'] = 'Europe/Zurich'
+	time.tzset()
 	# set alternate now for time related test cases:
 	MyTime.setAlternateNow(TEST_NOW)
 
@@ -292,17 +296,10 @@ old_TZ = os.environ.get('TZ', None)
 def setUpMyTime():
 	# Set the time to a fixed, known value
 	# Sun Aug 14 12:00:00 CEST 2005
-	# yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match
-	os.environ['TZ'] = 'Europe/Zurich'
-	time.tzset()
 	MyTime.setTime(TEST_NOW)
 
 
 def tearDownMyTime():
-	os.environ.pop('TZ')
-	if old_TZ: # pragma: no cover
-		os.environ['TZ'] = old_TZ
-	time.tzset()
 	MyTime.myTime = None
 
 

From a2cf34a64ef3348b284135e04659418d2a04ac5f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 17 Nov 2016 21:04:51 +0100
Subject: [PATCH 235/623] code review: added endpos to found tuple, just to be
 safe by unpack

---
 fail2ban/server/datedetector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 458f8d11..01cb43fe 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -359,7 +359,7 @@ class DateDetector(object):
 		if not match:
 			self.__lastTemplIdx = 0x7fffffff
 			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
-			found = None, 0x7fffffff, -1
+			found = None, 0x7fffffff, 0x7fffffff, -1
 			i = 0
 			for ddtempl in self.__templates:
 				if logSys.getEffectiveLevel() <= logLevel-1:

From 189e70d99cc38f09183c26b86cfd880039763314 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 18 Nov 2016 14:42:29 +0100
Subject: [PATCH 236/623] processLine etc. rewritten: - normalize calling
 parameters (persistent parameters moved from function arguments to filter
 member variables) - save last line as lambda instead of return it as string
 (lazy convert of process line tuple to string on demand, needed in
 fail2ban-regex only)

---
 fail2ban/client/fail2banregex.py  |  6 +++++-
 fail2ban/server/filter.py         | 28 +++++++++++++++++-----------
 fail2ban/tests/samplestestcase.py |  6 ++++--
 3 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 4096ac4e..a5aad76f 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -247,6 +247,9 @@ class Fail2banRegex(object):
 		self.raw = True if opts.raw else False
 		if opts.usedns:
 			self._filter.setUseDns(opts.usedns)
+		self._filter.returnRawHost = self.raw
+		self._filter.checkFindTime = False
+		self._filter.checkAllRegex = True
 
 	def decode_line(self, line):
 		return FileContainer.decode_line('<LOG>', self.encoding, line)
@@ -350,7 +353,8 @@ class Fail2banRegex(object):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
-			line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw)
+			ret = self._filter.processLine(line, date)
+			line = self._filter.processedLine()
 			for match in ret:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 9b1aefd6..2487b8f5 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -90,6 +90,12 @@ class Filter(JailThread):
 		## Error counter (protected, so can be used in filter implementations)
 		## if it reached 100 (at once), run-cycle will go idle
 		self._errors = 0
+		## return raw host (host is not dns):
+		self.returnRawHost = False
+		## check each regex (used for test purposes):
+		self.checkAllRegex = False
+		## if true ignores obsolete failures (failure time < now - findTime):
+		self.checkFindTime = True
 		## Ticks counter
 		self.ticks = 0
 
@@ -455,8 +461,7 @@ class Filter(JailThread):
 
 		return False
 
-	def processLine(self, line, date=None, returnRawHost=False,
-		checkAllRegex=False, checkFindTime=False):
+	def processLine(self, line, date=None):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
 		if date:
@@ -476,14 +481,15 @@ class Filter(JailThread):
 			else:
 				tupleLine = (l, "", "", None)
 
-		return "".join(tupleLine[::2]), self.findFailure(
-			tupleLine, date, returnRawHost, checkAllRegex, checkFindTime)
+		# save last line (lazy convert of process line tuple to string on demand):
+		self.processedLine = lambda: "".join(tupleLine[::2])
+		return self.findFailure(tupleLine, date)
 
 	def processLineAndAdd(self, line, date=None):
 		"""Processes the line for failures and populates failManager
 		"""
 		try:
-			for element in self.processLine(line, date, checkFindTime=True)[1]:
+			for element in self.processLine(line, date):
 				ip = element[1]
 				unixTime = element[2]
 				lines = element[3]
@@ -539,10 +545,10 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, tupleLine, date=None, returnRawHost=False,
-		checkAllRegex=False, checkFindTime=False):
+	def findFailure(self, tupleLine, date=None):
 		failList = list()
 
+		returnRawHost = self.returnRawHost
 		cidr = IPAddr.CIDR_UNSPEC
 		if self.__useDns == "raw":
 			returnRawHost = True
@@ -577,7 +583,7 @@ class Filter(JailThread):
 			timeText = self.__lastTimeText or "".join(tupleLine[::2])
 			date = self.__lastDate
 
-		if checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
+		if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
 			logSys.log(5, "Ignore line since time %s < %s - %s", 
 				date, MyTime.time(), self.getFindTime())
 			return failList
@@ -598,7 +604,7 @@ class Filter(JailThread):
 					# The ignoreregex matched. Remove ignored match.
 					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
 					logSys.log(7, "Matched ignoreregex and was ignored")
-					if not checkAllRegex:
+					if not self.checkAllRegex:
 						break
 					else:
 						continue
@@ -641,7 +647,7 @@ class Filter(JailThread):
 								ip = IPAddr(fid, IPAddr.CIDR_RAW)
 							failList.append([failRegexIndex, ip, date,
 								failRegex.getMatchedLines(), fail])
-							if not checkAllRegex:
+							if not self.checkAllRegex:
 								break
 						else:
 							ips = DNSUtils.textToIp(host, self.__useDns)
@@ -649,7 +655,7 @@ class Filter(JailThread):
 								for ip in ips:
 									failList.append([failRegexIndex, ip, date,
 										failRegex.getMatchedLines(), fail])
-								if not checkAllRegex:
+								if not self.checkAllRegex:
 									break
 					except RegexException as e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 31b1812e..55dffdb0 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -44,6 +44,9 @@ class FilterSamplesRegex(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		self.filter = Filter(None)
+		self.filter.returnRawHost = True
+		self.filter.checkAllRegex = True
+		self.filter.checkFindTime = False
 		self.filter.active = True
 
 		setUpMyTime()
@@ -111,8 +114,7 @@ def testSampleRegexsFactory(name, basedir):
 			else:
 				faildata = {}
 
-			ret = self.filter.processLine(
-				line, returnRawHost=True, checkAllRegex=True)[1]
+			ret = self.filter.processLine(line)
 			if not ret:
 				# Check line is flagged as none match
 				self.assertFalse(faildata.get('match', True),

From c442569b63028c573389a673f7473a4731d43567 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 21 Nov 2016 16:35:33 +0100
Subject: [PATCH 237/623] executeCmd: added possibility to select success
 return codes ignorecommand: both return codes (0, 1) are success codes now,
 so no errors will be logged + test cases extended to check this (and error
 case)

---
 fail2ban/server/action.py             |  4 ++--
 fail2ban/server/filter.py             |  3 ++-
 fail2ban/server/utils.py              | 12 +++++++-----
 fail2ban/tests/files/ignorecommand.py |  3 +++
 fail2ban/tests/filtertestcase.py      |  4 ++++
 5 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 976adb20..62aa51dc 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -584,7 +584,7 @@ class CommandAction(ActionBase):
 		return self.executeCmd(realCmd, self.timeout)
 
 	@staticmethod
-	def executeCmd(realCmd, timeout=60):
+	def executeCmd(realCmd, timeout=60, **kwargs):
 		"""Executes a command.
 
 		Parameters
@@ -613,6 +613,6 @@ class CommandAction(ActionBase):
 
 		_cmd_lock.acquire()
 		try:
-			return Utils.executeCmd(realCmd, timeout, shell=True, output=False)
+			return Utils.executeCmd(realCmd, timeout, shell=True, output=False, **kwargs)
 		finally:
 			_cmd_lock.release()
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 2487b8f5..d263c901 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -455,7 +455,8 @@ class Filter(JailThread):
 		if self.__ignoreCommand:
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
 			logSys.debug('ignore command: ' + command)
-			ret_ignore = CommandAction.executeCmd(command)
+			ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1))
+			ret_ignore = ret and ret_ignore == 0
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
 			return ret_ignore
 
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 6ed3d8fa..6d74d2db 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -110,7 +110,7 @@ class Utils():
 		return flags
 
 	@staticmethod
-	def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True):
+	def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, success_codes=(0,)):
 		"""Executes a command.
 
 		Parameters
@@ -178,7 +178,7 @@ class Utils():
 			if not popen:
 				return False if not output else (False, stdout, stderr, retcode)
 
-		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
+		std_level = logging.DEBUG if retcode in success_codes else logging.ERROR
 		# if we need output (to return or to log it): 
 		if output or std_level >= logSys.getEffectiveLevel():
 			# if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
@@ -208,8 +208,8 @@ class Utils():
 				popen.stderr.close()
 
 		success = False
-		if retcode == 0:
-			logSys.debug("%-.40s -- returned successfully", realCmd)
+		if retcode in success_codes:
+			logSys.debug("%-.40s -- returned successfully %i", realCmd, retcode)
 			success = True
 		elif retcode is None:
 			logSys.error("%-.40s -- unable to kill PID %i", realCmd, popen.pid)
@@ -223,7 +223,9 @@ class Utils():
 			logSys.error("%-.40s -- returned %i", realCmd, retcode)
 			if msg:
 				logSys.info("HINT on %i: %s", retcode, msg % locals())
-		return success if not output else (success, stdout, stderr, retcode)
+		if output:
+			return success, stdout, stderr, retcode
+		return success if len(success_codes) == 1 else (success, retcode)
 	
 	@staticmethod
 	def wait_for(cond, timeout, interval=None):
diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py
index 7011b51b..8c115006 100755
--- a/fail2ban/tests/files/ignorecommand.py
+++ b/fail2ban/tests/files/ignorecommand.py
@@ -1,5 +1,8 @@
 #!/usr/bin/env fail2ban-python
 import sys
+if len(sys.argv) != 2 or sys.argv[1] == "":
+  sys.stderr.write('usage: ignorecommand IP')
+  exit(10)
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
 exit(1)
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 2b57ce47..a6f491dd 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -377,6 +377,10 @@ class IgnoreIP(LogCaptureTestCase):
 		self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>"))
 		self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
 		self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
+		self.assertLogged("returned successfully 0", "returned successfully 1", all=True)
+		self.pruneLog()
+		self.assertFalse(self.filter.inIgnoreIPList(""))
+		self.assertLogged("usage: ignorecommand IP", "returned 10", all=True)
 
 	def testIgnoreCauseOK(self):
 		ip = "93.184.216.34"

From 701abfd250c7f2e68dd2d6f4490fec624ec8ade5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 21 Nov 2016 17:00:08 +0100
Subject: [PATCH 238/623] ChangeLog entry added + indentation fix (space-tab
 replacement)

---
 ChangeLog                             | 1 +
 fail2ban/tests/files/ignorecommand.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6d7ecf42..05a2e38f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,6 +28,7 @@ TODO: implementing of options resp. other tasks from PR #1346
 * Pyinotify-backend: stability fix for sporadically errors in multi-threaded
   environment (without lock)
 * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values
+* Misleading errors logged from ignorecommand in success case on retcode 1 (gh-1194)
 
 ### New Features
 * IPv6 support:
diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py
index 8c115006..da48ff91 100755
--- a/fail2ban/tests/files/ignorecommand.py
+++ b/fail2ban/tests/files/ignorecommand.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env fail2ban-python
 import sys
 if len(sys.argv) != 2 or sys.argv[1] == "":
-  sys.stderr.write('usage: ignorecommand IP')
-  exit(10)
+	sys.stderr.write('usage: ignorecommand IP')
+	exit(10)
 if sys.argv[1] == "10.0.0.1":
 	exit(0)
 exit(1)

From 261f875748cba31369bdedca5add04059b1c4a5c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 21 Nov 2016 19:00:53 +0100
Subject: [PATCH 239/623] Fixed sporadic tab-replacement (`\n\t` instead of `\n
 ` by word wrapping) in mime content of smtp-message in test cases, see
 https://github.com/fail2ban/fail2ban/pull/1410#issuecomment-262000804

---
 fail2ban/tests/action_d/test_smtp.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 5c8b1923..bfbf7b61 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -21,6 +21,7 @@ import os
 import smtpd
 import threading
 import unittest
+import re
 import sys
 if sys.version_info >= (3, 3):
 	import importlib
@@ -38,7 +39,9 @@ class TestSMTPServer(smtpd.SMTPServer):
 		self.peer = peer
 		self.mailfrom = mailfrom
 		self.rcpttos = rcpttos
-		self.data = data
+		self.org_data = data
+		# replace new line (with tab or space) for possible mime translations (word wrap):
+		self.data = re.sub(r"\n[\t ]", " ", data)
 
 
 class SMTPActionTest(unittest.TestCase):
@@ -104,9 +107,9 @@ class SMTPActionTest(unittest.TestCase):
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		subject = "Subject: [Fail2Ban] %s: banned %s" % (
 			self.jail.name, aInfo['ip'])
-		self.assertIn(subject, self.smtpd.data.replace("\n", ""))
-		self.assertTrue(
-			"%i attempts" % aInfo['failures'] in self.smtpd.data)
+		self.assertIn(subject, self.smtpd.data)
+		self.assertIn(
+			"%i attempts" % aInfo['failures'], self.smtpd.data)
 
 		self.action.matches = "matches"
 		self.action.ban(aInfo)

From 528a7a5abbf388def344387cd77f8f2fb54b532d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 11:14:27 +0100
Subject: [PATCH 240/623] systemd service update: - starting service in normal
 mode (without forking) - does not restart if service exited normally
 (exit-code 0, e.g. stopped via fail2ban-client) - does not restart if service
 can not start (exit-code 255, e.g. wrong configuration, etc.) - service can
 be additionally started/stopped with commands (fail2ban-client,
 fail2ban-server)

---
 files/fail2ban.service | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/files/fail2ban.service b/files/fail2ban.service
index e79faed1..5b661100 100644
--- a/files/fail2ban.service
+++ b/files/fail2ban.service
@@ -5,12 +5,13 @@ After=network.target iptables.service firewalld.service
 PartOf=iptables.service firewalld.service
 
 [Service]
-Type=forking
-ExecStart=/usr/bin/fail2ban-client -x start
+Type=simple
+ExecStart=/usr/bin/fail2ban-server -xf start
 ExecStop=/usr/bin/fail2ban-client stop
 ExecReload=/usr/bin/fail2ban-client reload
 PIDFile=/var/run/fail2ban/fail2ban.pid
-Restart=always
+Restart=on-failure
+RestartPreventExitStatus=0 255
 
 [Install]
 WantedBy=multi-user.target

From 77dc5a334c0e1ba79482a7799fefef27b6df413e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 13:19:09 +0100
Subject: [PATCH 241/623] really skips invalid jails (because of theirs wrong
 configuration) - server starts nevertheless, as long as one jail was
 successful configured; message about wrong jail configuration logged in
 client log (stdout, systemd journal etc.) and in server log as error

---
 fail2ban/client/jailreader.py  | 134 ++++++++++++++++++---------------
 fail2ban/client/jailsreader.py |  13 +++-
 fail2ban/server/transmitter.py |   3 +
 3 files changed, 86 insertions(+), 64 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 9d01a693..df6c6664 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -118,70 +118,78 @@ class JailReader(ConfigReader):
 		defsec = self._cfg.get_defaults()
 		defsec["fail2ban_version"] = version
 
-		# Read first options only needed for merge defaults ('known/...' from filter):
-		self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True)
-		if not self.__opts:
-			return False
-		
-		if self.isEnabled():
-			# Read filter
-			if self.__opts["filter"]:
-				filterName, filterOpt = JailReader.extractOptions(
-					self.__opts["filter"])
-				self.__filter = FilterReader(
-					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
-				ret = self.__filter.read()
-				# merge options from filter as 'known/...':
-				self.__filter.getOptions(self.__opts)
-				ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
-				if not ret:
-					logSys.error("Unable to read the filter")
-					return False
-			else:
-				self.__filter = None
-				logSys.warning("No filter set for jail %s" % self.__name)
+		try:
 
-			# Read second all options (so variables like %(known/param) can be interpolated):
-			self.__opts = ConfigReader.getOptions(self, self.__name, opts)
+			# Read first options only needed for merge defaults ('known/...' from filter):
+			self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True)
 			if not self.__opts:
-				return False
-		
-			# cumulate filter options again (ignore given in jail):
-			if self.__filter:
-				self.__filter.getOptions(self.__opts)
-		
-			# Read action
-			for act in self.__opts["action"].split('\n'):
-				try:
-					if not act:			  # skip empty actions
-						continue
-					actName, actOpt = JailReader.extractOptions(act)
-					if actName.endswith(".py"):
-						self.__actions.append([
-							"set",
-							self.__name,
-							"addaction",
-							actOpt.pop("actname", os.path.splitext(actName)[0]),
-							os.path.join(
-								self.getBaseDir(), "action.d", actName),
-							json.dumps(actOpt),
-							])
-					else:
-						action = ActionReader(
-							actName, self.__name, actOpt,
-							share_config=self.share_config, basedir=self.getBaseDir())
-						ret = action.read()
-						if ret:
-							action.getOptions(self.__opts)
-							self.__actions.append(action)
+				raise ValueError("Init jail options failed")
+			
+			if self.isEnabled():
+				# Read filter
+				if self.__opts["filter"]:
+					filterName, filterOpt = JailReader.extractOptions(
+						self.__opts["filter"])
+					self.__filter = FilterReader(
+						filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
+					ret = self.__filter.read()
+					# merge options from filter as 'known/...':
+					self.__filter.getOptions(self.__opts)
+					ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
+					if not ret:
+						raise ValueError("Unable to read the filter %r" % filterName)
+				else:
+					self.__filter = None
+					logSys.warning("No filter set for jail %s" % self.__name)
+
+				# Read second all options (so variables like %(known/param) can be interpolated):
+				self.__opts = ConfigReader.getOptions(self, self.__name, opts)
+				if not self.__opts:
+					raise ValueError("Read jail options failed")
+			
+				# cumulate filter options again (ignore given in jail):
+				if self.__filter:
+					self.__filter.getOptions(self.__opts)
+			
+				# Read action
+				for act in self.__opts["action"].split('\n'):
+					try:
+						if not act:			  # skip empty actions
+							continue
+						actName, actOpt = JailReader.extractOptions(act)
+						if actName.endswith(".py"):
+							self.__actions.append([
+								"set",
+								self.__name,
+								"addaction",
+								actOpt.pop("actname", os.path.splitext(actName)[0]),
+								os.path.join(
+									self.getBaseDir(), "action.d", actName),
+								json.dumps(actOpt),
+								])
 						else:
-							raise AttributeError("Unable to read action")
-				except Exception as e:
-					logSys.error("Error in action definition " + act)
-					logSys.debug("Caught exception: %s" % (e,))
-					return False
-			if not len(self.__actions):
-				logSys.warning("No actions were defined for %s" % self.__name)
+							action = ActionReader(
+								actName, self.__name, actOpt,
+								share_config=self.share_config, basedir=self.getBaseDir())
+							ret = action.read()
+							if ret:
+								action.getOptions(self.__opts)
+								self.__actions.append(action)
+							else:
+								raise AttributeError("Unable to read action")
+					except Exception as e:
+						logSys.debug("Caught exception: %s" % (e,))
+						raise ValueError("Error in action definition %r" % e)
+				if not len(self.__actions):
+					logSys.warning("No actions were defined for %s" % self.__name)
+			
+		except ValueError as e:
+			e = str(e)
+			logSys.error(e)
+			if not self.__opts:
+				self.__opts = dict()
+			self.__opts['config-error'] = e
+			return False
 		return True
 	
 	def convert(self, allow_no_files=False):
@@ -195,6 +203,10 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
+		e = self.__opts.get('config-error')
+		if e:
+			stream.extend([['config-error', "Jail '%s' skipped, because of wrong configuration: %s" % (self.__name, e)]])
+			return stream
 		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					not self.__opts.get('backend', None).startswith("systemd"):
diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py
index 09725ec9..ec7baca7 100644
--- a/fail2ban/client/jailsreader.py
+++ b/fail2ban/client/jailsreader.py
@@ -66,7 +66,7 @@ class JailsReader(ConfigReader):
 			sections = [ section ]
 
 		# Get the options of all jails.
-		parse_status = True
+		parse_status = None
 		for sec in sections:
 			if sec == 'INCLUDES':
 				continue
@@ -77,11 +77,17 @@ class JailsReader(ConfigReader):
 			ret = jail.getOptions()
 			if ret:
 				if jail.isEnabled():
+					# at least one jail was successful:
+					parse_status = True
 					# We only add enabled jails
 					self.__jails.append(jail)
 			else:
 				logSys.error("Errors in jail %r. Skipping..." % sec)
-				parse_status = False
+				self.__jails.append(jail)
+				if parse_status is None:
+					parse_status = False
+		if parse_status is None:
+			parse_status = True
 		return parse_status
 
 	def convert(self, allow_no_files=False):
@@ -103,7 +109,8 @@ class JailsReader(ConfigReader):
 			stream.extend(jail.convert(allow_no_files=allow_no_files))
 		# Start jails
 		for jail in self.__jails:
-			stream.append(["start", jail.getName()])
+			if not jail.options.get('config-error'):
+				stream.append(["start", jail.getName()])
 
 		return stream
 
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 2f5be043..ae1017b9 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -131,6 +131,9 @@ class Transmitter:
 			return self.status(command[1:])
 		elif command[0] == "version":
 			return version.version
+		elif command[0] == "config-error":
+			logSys.error(command[1])
+			return None
 		raise Exception("Invalid command")
 	
 	def __commandSet(self, command, multiple=False):

From c6e8c700f7f712d9b5c93c125bdcefbef670c7ad Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 13:57:06 +0100
Subject: [PATCH 242/623] test cases fixed

---
 fail2ban/client/jailreader.py          | 14 +++++++++-----
 fail2ban/tests/clientreadertestcase.py | 17 +++++++++--------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index df6c6664..74072481 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -127,9 +127,11 @@ class JailReader(ConfigReader):
 			
 			if self.isEnabled():
 				# Read filter
-				if self.__opts["filter"]:
-					filterName, filterOpt = JailReader.extractOptions(
-						self.__opts["filter"])
+				flt = self.__opts["filter"]
+				if flt:
+					filterName, filterOpt = JailReader.extractOptions(flt)
+					if not filterName:
+						raise ValueError("Invalid filter declaration %r" % flt)
 					self.__filter = FilterReader(
 						filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
 					ret = self.__filter.read()
@@ -157,6 +159,8 @@ class JailReader(ConfigReader):
 						if not act:			  # skip empty actions
 							continue
 						actName, actOpt = JailReader.extractOptions(act)
+						if not actName:
+							raise ValueError("Invalid action declaration %r" % act)
 						if actName.endswith(".py"):
 							self.__actions.append([
 								"set",
@@ -178,8 +182,8 @@ class JailReader(ConfigReader):
 							else:
 								raise AttributeError("Unable to read action")
 					except Exception as e:
-						logSys.debug("Caught exception: %s" % (e,))
-						raise ValueError("Error in action definition %r" % e)
+						logSys.debug("Caught exception: %s", e, exc_info=True)
+						raise ValueError("Error in action definition %r: %r" % (act, e))
 				if not len(self.__actions):
 					logSys.warning("No actions were defined for %s" % self.__name)
 			
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index e68523c2..20d6ced1 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -193,13 +193,9 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertTrue(jail.read())
 		self.assertFalse(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
-		self.assertLogged('Error in action definition joho[foo')
-		# This unittest has been deactivated for some time...
-		# self.assertLogged(
-		#     'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')
-		#   let's test for what is actually logged and handle changes in the future
+		self.assertLogged("Error in action definition 'joho[foo'")
 		self.assertLogged(
-			"Caught exception: 'NoneType' object has no attribute 'endswith'")
+			"Caught exception: Invalid action declaration 'joho[foo'")
 
 	if STOCK:
 		def testStockSSHJail(self):
@@ -496,7 +492,7 @@ class JailsReaderTest(LogCaptureTestCase):
 	def testReadTestJailConf(self):
 		jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jails.read())
-		self.assertFalse(jails.getOptions())
+		self.assertTrue(jails.getOptions())
 		self.assertRaises(ValueError, jails.convert)
 		comm_commands = jails.convert(allow_no_files=True)
 		self.maxDiff = None
@@ -525,7 +521,12 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['start', 'emptyaction'],
 			 ['start', 'missinglogfiles'],
 			 ['start', 'brokenaction'],
-			 ['start', 'parse_to_end_of_jail.conf'],]))
+			 ['start', 'parse_to_end_of_jail.conf'],
+			 ['config-error',
+				'Jail \'brokenactiondef\' skipped, because of wrong configuration: Error in action definition \'joho[foo\': ValueError("Invalid action declaration \'joho[foo\'",)'],
+			 ['config-error',
+				"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
+			 ]))
 		self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...")
 		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
 

From e52b47d8f52b9dd1f5704b62f21b3bf9c6c604b5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 13:57:20 +0100
Subject: [PATCH 243/623] normalized log output (all jail parameters in filter
 are indented with 2 spaces)

---
 fail2ban/server/actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index dcfe09f6..dd4c97c2 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -193,7 +193,7 @@ class Actions(JailThread, Mapping):
 	def setBanTime(self, value):
 		value = MyTime.str2seconds(value)
 		self.__banManager.setBanTime(value)
-		logSys.info("Set banTime = %s" % value)
+		logSys.info("  banTime: %s" % value)
 	
 	##
 	# Get the ban time.

From 4882093a41c031d79e3aeace447d762009f6528c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 14:08:35 +0100
Subject: [PATCH 244/623] test cases extended: cover skipping invalid jail

---
 fail2ban/tests/fail2banclienttestcase.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e212ff23..cec48cc3 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -768,6 +768,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_action_cfg(actname="test-action2")
 
 		_write_jail_cfg(enabled=[1], actions=[1,2])
+		# append one wrong configured jail:
+		_write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", 
+			"", "filter = broken-jail-filter", "enabled = true")
+
 		_write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3))
 		_write_file(test2log, "w")
 		_write_file(test3log, "w")
@@ -786,6 +790,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action1: ** start'", 
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
+
+		# broken jail was logged (in client and server log):
+		self.assertLogged(
+			"Unable to read the filter 'broken-jail-filter'",
+			"Errors in jail 'broken-jail'. Skipping...",
+			"Jail 'broken-jail' skipped, because of wrong configuration", all=True)
 		
 		# enable both jails, 3 logs for jail1, etc...
 		# truncate test-log - we should not find unban/ban again by reload:

From 3e9852d4d2a7c73936a4cbaafd17bf13a0e412a6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 14:56:54 +0100
Subject: [PATCH 245/623] code review, increase coverage

---
 fail2ban/client/jailreader.py          | 24 +++++++++++++++---------
 fail2ban/client/jailsreader.py         | 10 ++--------
 fail2ban/tests/clientreadertestcase.py | 18 ++++++++++++++----
 fail2ban/tests/config/jail.conf        |  8 ++++++++
 4 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 74072481..b63df5f1 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -122,8 +122,8 @@ class JailReader(ConfigReader):
 
 			# Read first options only needed for merge defaults ('known/...' from filter):
 			self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True)
-			if not self.__opts:
-				raise ValueError("Init jail options failed")
+			if not self.__opts: # pragma: no cover
+				raise JailDefError("Init jail options failed")
 			
 			if self.isEnabled():
 				# Read filter
@@ -131,7 +131,7 @@ class JailReader(ConfigReader):
 				if flt:
 					filterName, filterOpt = JailReader.extractOptions(flt)
 					if not filterName:
-						raise ValueError("Invalid filter declaration %r" % flt)
+						raise JailDefError("Invalid filter definition %r" % flt)
 					self.__filter = FilterReader(
 						filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
 					ret = self.__filter.read()
@@ -139,15 +139,15 @@ class JailReader(ConfigReader):
 					self.__filter.getOptions(self.__opts)
 					ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
 					if not ret:
-						raise ValueError("Unable to read the filter %r" % filterName)
+						raise JailDefError("Unable to read the filter %r" % filterName)
 				else:
 					self.__filter = None
 					logSys.warning("No filter set for jail %s" % self.__name)
 
 				# Read second all options (so variables like %(known/param) can be interpolated):
 				self.__opts = ConfigReader.getOptions(self, self.__name, opts)
-				if not self.__opts:
-					raise ValueError("Read jail options failed")
+				if not self.__opts: # pragma: no cover
+					raise JailDefError("Read jail options failed")
 			
 				# cumulate filter options again (ignore given in jail):
 				if self.__filter:
@@ -160,7 +160,7 @@ class JailReader(ConfigReader):
 							continue
 						actName, actOpt = JailReader.extractOptions(act)
 						if not actName:
-							raise ValueError("Invalid action declaration %r" % act)
+							raise JailDefError("Invalid action definition %r" % act)
 						if actName.endswith(".py"):
 							self.__actions.append([
 								"set",
@@ -180,14 +180,16 @@ class JailReader(ConfigReader):
 								action.getOptions(self.__opts)
 								self.__actions.append(action)
 							else:
-								raise AttributeError("Unable to read action")
+								raise JailDefError("Unable to read action %r" % actName)
+					except JailDefError:
+						raise
 					except Exception as e:
 						logSys.debug("Caught exception: %s", e, exc_info=True)
 						raise ValueError("Error in action definition %r: %r" % (act, e))
 				if not len(self.__actions):
 					logSys.warning("No actions were defined for %s" % self.__name)
 			
-		except ValueError as e:
+		except JailDefError as e:
 			e = str(e)
 			logSys.error(e)
 			if not self.__opts:
@@ -280,3 +282,7 @@ class JailReader(ConfigReader):
 					val for val in optmatch.group(2,3,4) if val is not None][0]
 				option_opts[opt.strip()] = value.strip()
 		return option_name, option_opts
+
+
+class JailDefError(Exception):
+	pass
diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py
index ec7baca7..7d81d0a0 100644
--- a/fail2ban/client/jailsreader.py
+++ b/fail2ban/client/jailsreader.py
@@ -84,11 +84,8 @@ class JailsReader(ConfigReader):
 			else:
 				logSys.error("Errors in jail %r. Skipping..." % sec)
 				self.__jails.append(jail)
-				if parse_status is None:
-					parse_status = False
-		if parse_status is None:
-			parse_status = True
-		return parse_status
+				if parse_status is None: parse_status = False
+		return True if parse_status != False else False
 
 	def convert(self, allow_no_files=False):
 		"""Convert read before __opts and jails to the commands stream
@@ -101,9 +98,6 @@ class JailsReader(ConfigReader):
 		"""
 
 		stream = list()
-		for opt in self.__opts:
-			if opt == "":
-				stream.append([])
 		# Convert jails
 		for jail in self.__jails:
 			stream.extend(jail.convert(allow_no_files=allow_no_files))
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 20d6ced1..33c217cd 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -193,9 +193,15 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertTrue(jail.read())
 		self.assertFalse(jail.getOptions())
 		self.assertTrue(jail.isEnabled())
-		self.assertLogged("Error in action definition 'joho[foo'")
-		self.assertLogged(
-			"Caught exception: Invalid action declaration 'joho[foo'")
+		self.assertLogged("Invalid action definition 'joho[foo'")
+
+	def testJailFilterBrokenDef(self):
+		jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG,
+			share_config=IMPERFECT_CONFIG_SHARE_CFG)
+		self.assertTrue(jail.read())
+		self.assertFalse(jail.getOptions())
+		self.assertTrue(jail.isEnabled())
+		self.assertLogged("Invalid filter definition 'flt[test'")
 
 	if STOCK:
 		def testStockSSHJail(self):
@@ -523,7 +529,11 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['start', 'brokenaction'],
 			 ['start', 'parse_to_end_of_jail.conf'],
 			 ['config-error',
-				'Jail \'brokenactiondef\' skipped, because of wrong configuration: Error in action definition \'joho[foo\': ValueError("Invalid action declaration \'joho[foo\'",)'],
+				"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"],
+			 ['config-error',
+				"Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"],
+			 ['config-error',
+				"Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"],
 			 ['config-error',
 				"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
 			 ]))
diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf
index bf1dea45..659e3fd3 100644
--- a/fail2ban/tests/config/jail.conf
+++ b/fail2ban/tests/config/jail.conf
@@ -27,10 +27,18 @@ logpath = /weapons/of/mass/destruction
 enabled = true
 action = joho[foo
 
+[brokenfilterdef]
+enabled = true
+filter = flt[test
+
 [brokenaction]
 enabled = true
 action = brokenaction
 
+[missingaction]
+enabled = true
+action = noactionfileforthisaction
+
 [missingbitsjail]
 enabled = true
 filter = catchallthebadies

From fdac44ca589ff5d9de8afde3865022dc7f702858 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 17:08:44 +0100
Subject: [PATCH 246/623] introduced new option `-t` or `--test` to test
 configuration resp. start server only if configuration is clean (not skip
 wrong configured jails if option `-t` specified);

---
 fail2ban/client/configurator.py          |  4 +--
 fail2ban/client/fail2bancmdline.py       | 33 ++++++++++++++++----
 fail2ban/client/fail2banserver.py        | 38 ++++++++++++------------
 fail2ban/client/jailsreader.py           | 13 ++++----
 fail2ban/tests/fail2banclienttestcase.py | 30 +++++++++++++++++++
 5 files changed, 86 insertions(+), 32 deletions(-)

diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py
index 4d28238f..e8472ac1 100644
--- a/fail2ban/client/configurator.py
+++ b/fail2ban/client/configurator.py
@@ -72,9 +72,9 @@ class Configurator:
 	def getEarlyOptions(self):
 		return self.__fail2ban.getEarlyOptions()
 
-	def getOptions(self, jail=None, updateMainOpt=None):
+	def getOptions(self, jail=None, updateMainOpt=None, ignoreWrong=True):
 		self.__fail2ban.getOptions(updateMainOpt)
-		return self.__jails.getOptions(jail)
+		return self.__jails.getOptions(jail, ignoreWrong=ignoreWrong)
 		
 	def convertToProtocol(self):
 		self.__streams["general"] = self.__fail2ban.convert()
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 74236ab1..7d0eeead 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -47,6 +47,7 @@ class Fail2banCmdLine():
 	def __init__(self):
 		self._argv = self._args = None
 		self._configurator = None
+		self.cleanConfOnly = False
 		self.resetConf()
 
 	def resetConf(self):
@@ -101,6 +102,7 @@ class Fail2banCmdLine():
 		output("    --logtarget <FILE>|STDOUT|STDERR|SYSLOG")
 		output("    --syslogsocket auto|<FILE>")
 		output("    -d                      dump configuration. For debugging")
+		output("    -t, --test              test configuration (can be also specified with start parameters)")
 		output("    -i                      interactive mode")
 		output("    -v                      increase verbosity")
 		output("    -q                      decrease verbosity")
@@ -136,6 +138,9 @@ class Fail2banCmdLine():
 				self._conf[ o[2:] ] = opt[1]
 			elif o == "-d":
 				self._conf["dump"] = True
+			elif o == "-t" or o == "--test":
+				self.cleanConfOnly = True
+				self._conf["test"] = True
 			elif o == "-v":
 				self._conf["verbose"] += 1
 			elif o == "-q":
@@ -173,8 +178,8 @@ class Fail2banCmdLine():
 
 			# Reads the command line options.
 			try:
-				cmdOpts = 'hc:s:p:xfbdviqV'
-				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'timeout=', 'help', 'version']
+				cmdOpts = 'hc:s:p:xfbdtviqV'
+				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'timeout=', 'help', 'version']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
@@ -225,13 +230,30 @@ class Fail2banCmdLine():
 			logSys.info("Using pid file %s, [%s] logging to %s",
 				self._conf["pidfile"], logging.getLevelName(llev), self._conf["logtarget"])
 
+			readcfg = True
 			if self._conf.get("dump", False):
-				ret, stream = self.readConfig()
+				if readcfg:
+					ret, stream = self.readConfig()
+					readcfg = False
 				self.dumpConfig(stream)
-				return ret
+				if not self._conf.get("test", False):
+					return ret
+
+			if self._conf.get("test", False):
+				if readcfg:
+					readcfg = False
+					ret, stream = self.readConfig()
+				if not ret:
+					raise ServerExecutionException("ERROR: test configuration failed")
+				# exit after test if no commands specified (test only):
+				if not len(self._args):
+					output("OK: configuration test is successful")
+					return ret
 
 			# Nothing to do here, process in client/server
 			return None
+		except ServerExecutionException:
+			raise
 		except Exception as e:
 			output("ERROR: %s" % (e,))
 			if verbose > 2:
@@ -246,7 +268,8 @@ class Fail2banCmdLine():
 		try:
 			self.configurator.Reload()
 			self.configurator.readAll()
-			ret = self.configurator.getOptions(jail, self._conf)
+			ret = self.configurator.getOptions(jail, self._conf, 
+				ignoreWrong=not self.cleanConfOnly)
 			self.configurator.convertToProtocol()
 			stream = self.configurator.getConfigStream()
 		except Exception as e:
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index dfee34d2..006a02cf 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -144,27 +144,27 @@ class Fail2banServer(Fail2banCmdLine):
 		return cli
 
 	def start(self, argv):
-		# Command line options
-		ret = self.initCmdLine(argv)
-		if ret is not None:
-			return ret
-
-		# Commands
-		args = self._args
-
-		cli = None
-		# Just start:
-		if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False):
-			pass
-		else:
-			# If client mode - whole processing over client:
-			if len(args) or self._conf.get("interactive", False):
-				cli = self._Fail2banClient()
-				return cli.start(argv)
-
-		# Start the server:
 		server = None
 		try:
+			# Command line options
+			ret = self.initCmdLine(argv)
+			if ret is not None:
+				return ret
+
+			# Commands
+			args = self._args
+
+			cli = None
+			# Just start:
+			if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False):
+				pass
+			else:
+				# If client mode - whole processing over client:
+				if len(args) or self._conf.get("interactive", False):
+					cli = self._Fail2banClient()
+					return cli.start(argv)
+
+			# Start the server:
 			from ..server.utils import Utils
 			# background = True, if should be new process running in background, otherwise start in foreground
 			# process will be forked in daemonize, inside of Server module.
diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py
index 7d81d0a0..cd3409b4 100644
--- a/fail2ban/client/jailsreader.py
+++ b/fail2ban/client/jailsreader.py
@@ -54,7 +54,7 @@ class JailsReader(ConfigReader):
 		self.__jails = list()
 		return ConfigReader.read(self, "jail")
 
-	def getOptions(self, section=None):
+	def getOptions(self, section=None, ignoreWrong=True):
 		"""Reads configuration for jail(s) and adds enabled jails to __jails
 		"""
 		opts = []
@@ -66,7 +66,7 @@ class JailsReader(ConfigReader):
 			sections = [ section ]
 
 		# Get the options of all jails.
-		parse_status = None
+		parse_status = 0
 		for sec in sections:
 			if sec == 'INCLUDES':
 				continue
@@ -78,14 +78,15 @@ class JailsReader(ConfigReader):
 			if ret:
 				if jail.isEnabled():
 					# at least one jail was successful:
-					parse_status = True
+					parse_status |= 1
 					# We only add enabled jails
 					self.__jails.append(jail)
 			else:
-				logSys.error("Errors in jail %r. Skipping..." % sec)
+				logSys.error("Errors in jail %r.%s", sec, " Skipping..." if ignoreWrong else "")
 				self.__jails.append(jail)
-				if parse_status is None: parse_status = False
-		return True if parse_status != False else False
+				# at least one jail was invalid:
+				parse_status |= 2
+		return ((ignoreWrong and parse_status & 1) or not (parse_status & 2))
 
 	def convert(self, allow_no_files=False):
 		"""Convert read before __opts and jails to the commands stream
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index cec48cc3..e68d9779 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -675,6 +675,36 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))
 
+	@with_tmpdir
+	@with_kill_srv
+	def testServerTestFailStart(self, tmp):
+		# started directly here, so prevent overwrite test cases logger with "INHERITED"
+		startparams = _start_params(tmp, logtarget="INHERITED")
+		cfg = pjoin(tmp, "config")
+
+		# test configuration is correct:
+		self.pruneLog("[test-phase 0]")
+		self.execSuccess(startparams, "--test")
+		self.assertLogged("OK: configuration test is successful")
+
+		# append one wrong configured jail:
+		_write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", 
+			"", "filter = broken-jail-filter", "enabled = true")
+
+		# first try test config:
+		self.pruneLog("[test-phase 0a]")
+		self.execFailed(startparams, "--test")
+		self.assertLogged("Unable to read the filter 'broken-jail-filter'",
+			"Errors in jail 'broken-jail'.",
+			"ERROR: test configuration failed", all=True)
+
+		# failed to start with test config:
+		self.pruneLog("[test-phase 0b]")
+		self.execFailed(startparams, "-t", "start")
+		self.assertLogged("Unable to read the filter 'broken-jail-filter'",
+			"Errors in jail 'broken-jail'.",
+			"ERROR: test configuration failed", all=True)
+
 	@with_tmpdir
 	def testKillAfterStart(self, tmp):
 		try:

From 8ed5b44bfd2256f8ffe8ad867b7cd9fc7cc0e467 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 17:38:32 +0100
Subject: [PATCH 247/623] no cover for sporadic executed (time-related) code
 pieces (just to prevent randomly increasing/decreasing of coverage)

---
 fail2ban/client/fail2banclient.py | 2 +-
 fail2ban/server/asyncserver.py    | 2 +-
 fail2ban/server/utils.py          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
index 007afd57..0a1ae4f1 100755
--- a/fail2ban/client/fail2banclient.py
+++ b/fail2ban/client/fail2banclient.py
@@ -125,7 +125,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			if client:
 				try :
 					client.close()
-				except Exception as e:
+				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
 						logSys.debug(e)
 			if showRet or c[0] == 'echo':
diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index d1818d7a..9cc74658 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -241,7 +241,7 @@ class AsyncServer(asyncore.dispatcher):
 	def _remove_sock(self):
 		try:
 			os.remove(self.__sock)
-		except OSError as e:
+		except OSError as e: # pragma: no cover
 			if e.errno != errno.ENOENT:
 				raise
 
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 6d74d2db..57da495a 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -170,7 +170,7 @@ class Utils():
 					time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
 					retcode = popen.poll()
 					#logSys.debug("%s -- killed %s ", realCmd, retcode)
-				if retcode is None and not Utils.pid_exists(pgid):
+				if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover
 					retcode = signal.SIGKILL
 		except OSError as e:
 			stderr = "%s -- failed with %s" % (realCmd, e)

From 7256a5cb8e0c37bca69fd684bd3e568a4cc46540 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 22 Nov 2016 17:55:27 +0100
Subject: [PATCH 248/623] code review: back to previous code - no skipping in
 testReadTestJailConf

---
 fail2ban/tests/clientreadertestcase.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 33c217cd..5129df61 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -498,7 +498,7 @@ class JailsReaderTest(LogCaptureTestCase):
 	def testReadTestJailConf(self):
 		jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jails.read())
-		self.assertTrue(jails.getOptions())
+		self.assertFalse(jails.getOptions(ignoreWrong=False))
 		self.assertRaises(ValueError, jails.convert)
 		comm_commands = jails.convert(allow_no_files=True)
 		self.maxDiff = None
@@ -537,7 +537,8 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['config-error',
 				"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
 			 ]))
-		self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...")
+		self.assertLogged("Errors in jail 'missingbitsjail'.")
+		self.assertNotLogged("Skipping...")
 		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
 
 	if STOCK:

From 1cd67ecaa260bdaa6e62383ed783d5910587c258 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 23 Nov 2016 22:03:12 +0100
Subject: [PATCH 249/623] automatically creates /var/run/fail2ban before start
 fail2ban (systems which /var/run/ is virtual resp. memory mount device)

---
 files/fail2ban.service | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/fail2ban.service b/files/fail2ban.service
index 5b661100..1ef7b5e6 100644
--- a/files/fail2ban.service
+++ b/files/fail2ban.service
@@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service
 
 [Service]
 Type=simple
+ExecStartPre=/bin/mkdir -p /var/run/fail2ban
 ExecStart=/usr/bin/fail2ban-server -xf start
 ExecStop=/usr/bin/fail2ban-client stop
 ExecReload=/usr/bin/fail2ban-client reload

From 45174c5eaf1479a2b5fec594dce421364b5cfd36 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 24 Nov 2016 12:13:47 +0100
Subject: [PATCH 250/623] if fail2ban running as systemd-service, for logging
 to the systemd-journal, the `logtarget` could be set to STDOUT small fixes by
 logging in stdout (+ system targets also allowed in lowercase now)

---
 fail2ban/client/fail2bancmdline.py |  1 +
 fail2ban/server/server.py          | 10 ++++++----
 files/fail2ban.service             |  2 ++
 man/fail2ban-server.1              |  3 +++
 man/jail.conf.5                    |  4 +++-
 5 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 74236ab1..c7c40120 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -274,6 +274,7 @@ class Fail2banCmdLine():
 	def exit(code=0):
 		logSys.debug("Exit with code %s", code)
 		# because of possible buffered output in python, we should flush it before exit:
+		logging.shutdown()
 		sys.stdout.flush()
 		sys.stderr.flush()
 		# exit
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index efd47b09..313b6ee5 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -530,17 +530,19 @@ class Server:
 	# @param target the logging target
 	
 	def setLogTarget(self, target):
+		# check reserved targets in uppercase, don't change target, because it can be file:
+		systarget = target.upper()
 		with self.__loggingLock:
 			# don't set new handlers if already the same
 			# or if "INHERITED" (foreground worker of the test cases, to prevent stop logging):
 			if self.__logTarget == target:
 				return True
-			if target == "INHERITED":
+			if systarget == "INHERITED":
 				self.__logTarget = target
 				return True
 			# set a format which is simpler for console use
 			fmt = "%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
-			if target == "SYSLOG":
+			if systarget == "SYSLOG":
 				# Syslog daemons already add date to the message.
 				fmt = "%(name)s[%(process)d]: %(levelname)s %(message)s"
 				facility = logging.handlers.SysLogHandler.LOG_DAEMON
@@ -559,9 +561,9 @@ class Server:
 						"Syslog socket file: %s does not exists"
 						" or is not a socket" % self.__syslogSocket)
 					return False
-			elif target == "STDOUT":
+			elif systarget == "STDOUT":
 				hdlr = logging.StreamHandler(sys.stdout)
-			elif target == "STDERR":
+			elif systarget == "STDERR":
 				hdlr = logging.StreamHandler(sys.stderr)
 			else:
 				# Target should be a file
diff --git a/files/fail2ban.service b/files/fail2ban.service
index 1ef7b5e6..6eeba957 100644
--- a/files/fail2ban.service
+++ b/files/fail2ban.service
@@ -8,6 +8,8 @@ PartOf=iptables.service firewalld.service
 Type=simple
 ExecStartPre=/bin/mkdir -p /var/run/fail2ban
 ExecStart=/usr/bin/fail2ban-server -xf start
+# if should be logged in systemd journal, use following line or set logtarget to stdout in fail2ban.local
+# ExecStart=/usr/bin/fail2ban-server -xf --logtarget=stdout start
 ExecStop=/usr/bin/fail2ban-client stop
 ExecReload=/usr/bin/fail2ban-client reload
 PIDFile=/var/run/fail2ban/fail2ban.pid
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index de8ba6a4..09dcd65a 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -23,6 +23,9 @@ pidfile path
 logging level
 .HP
 \fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
+logging target
+.br
+Note. If fail2ban running as systemd-service, for logging to the systemd-journal, the logtarget could be set to STDOUT
 .HP
 \fB\-\-syslogsocket\fR auto|<FILE>
 .TP
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 51a00cdc..2e333e5a 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -130,7 +130,9 @@ The items that can be set are:
 verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40)
 .TP
 .B logtarget
-log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR
+log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local
+.br
+Note. If fail2ban running as systemd-service, for logging to the systemd-journal, the logtarget could be set to STDOUT
 .br
 Only a single log target can be specified.
 If you change logtarget from the default value and you are using logrotate -- also adjust or disable rotation in the

From d908688b565fbc55fd63c1cf4ac305379afdfdca Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 24 Nov 2016 20:25:08 +0100
Subject: [PATCH 251/623] ChangeLog update

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 05a2e38f..ef2ce050 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -55,6 +55,8 @@ TODO: implementing of options resp. other tasks from PR #1346
     banned in this jail, if option `--unban` specified
   - `unban --all` - unbans all IP addresses (in all jails and database)
   - `unban <IP> ... <IP>` - unbans \<IP\> (in all jails and database) (see gh-1388)
+  - introduced new option `-t` or `--test` to test configuration resp. start server only 
+    if configuration is clean (fails by wrong configured jails if option `-t` specified)
 * New command action parameter `actionrepair` - command executed in order to restore
   sane environment in error case of `actioncheck`.
 
@@ -119,6 +121,10 @@ fail2ban-client set loglevel INFO
   - new replacement for `<ADDR>` in opposition to `<HOST>`, for separate
     usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6`
     together, without host (dns)
+* Misconfigured jails don't prevent fail2ban from starting, server starts 
+  nevertheless, as long as one jail was successful configured (gh-1619)
+  Message about wrong jail configuration logged in client log (stdout, systemd
+  journal etc.) and in server log with error level
 * fail2ban-testcases:
   - `assertLogged` extended with parameter wait (to wait up to specified timeout,
     before we throw assert exception) + test cases rewritten using that

From 308bba448ca7edfd46115b16d27ce18b2f2999c9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 24 Nov 2016 20:43:55 +0100
Subject: [PATCH 252/623] ChangeLog update

---
 ChangeLog | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 05a2e38f..b07d2842 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,6 +29,16 @@ TODO: implementing of options resp. other tasks from PR #1346
   environment (without lock)
 * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values
 * Misleading errors logged from ignorecommand in success case on retcode 1 (gh-1194)
+* fail2ban.service - systemd service updated (gh-1618):
+  - starting service in normal mode (without forking)
+  - does not restart if service exited normally (exit-code 0, e.g. stopped via fail2ban-client)
+  - does not restart if service can not start (exit-code 255, e.g. wrong configuration, etc.)
+  - service can be additionally started/stopped with commands (fail2ban-client, fail2ban-server)
+  - automatically creates `/var/run/fail2ban` directory before start fail2ban 
+    (systems with virtual resp. memory-based FS for `/var/run`), see gh-1531
+  - if fail2ban running as systemd-service, for logging to the systemd-journal, 
+    the `logtarget` could be set to STDOUT
+  - value `logtarget` for system targets allowed also in lowercase (stdout, stderr, syslog, etc.)
 
 ### New Features
 * IPv6 support:

From 097970781cbe61160824c834ca5cb4a065c2a8f8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 25 Nov 2016 16:53:46 +0100
Subject: [PATCH 253/623] filter/action (and its includes): substitution
 `%(param)s` may be used now (instead of `<param>`) for init-values specified
 in jail-configs via `action[param1="...", param2=...]`; substitution
 `<param>` should be used for dynamic interpolation only (todo: review
 configurations to replace it);

---
 fail2ban/client/configparserinc.py |  2 +-
 fail2ban/client/configreader.py    | 32 ++++++++++++++++++++++++------
 fail2ban/client/filterreader.py    |  6 ++++--
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index d6cda7f7..35fa7498 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -29,7 +29,7 @@ import re
 import sys
 from ..helpers import getLogger
 
-if sys.version_info >= (3,2): # pragma: no cover
+if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, \
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 643cdf3a..caab67aa 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -28,13 +28,25 @@ import glob
 import os
 from ConfigParser import NoOptionError, NoSectionError
 
-from .configparserinc import SafeConfigParserWithIncludes, logLevel
+from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
+# if sys.version_info >= (3,5):
+# 	def _merge_dicts(x, y):
+# 		return {**x, **y}
+# else:
+def _merge_dicts(x, y):
+	r = x
+	if y:
+		r = x.copy()
+		r.update(y)
+	return r
+
+
 class ConfigReader():
 	"""Generic config reader class.
 
@@ -127,9 +139,9 @@ class ConfigReader():
 			return self._cfg.options(*args)
 		return {}
 
-	def get(self, sec, opt):
+	def get(self, sec, opt, raw=False, vars={}):
 		if self._cfg is not None:
-			return self._cfg.get(sec, opt)
+			return self._cfg.get(sec, opt, raw, vars)
 		return None
 
 	def getOptions(self, *args, **kwargs):
@@ -210,6 +222,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	
 	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 		values = dict()
+		if pOptions is None:
+			pOptions = {}
 		for optname in options:
 			if isinstance(options, (list,tuple)):
 				if len(optname) > 2:
@@ -218,15 +232,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 					(opttype, optname), optvalue = optname, None
 			else:
 				opttype, optvalue = options[optname]
+			if optname in pOptions:
+				continue
 			try:
 				if opttype == "bool":
 					v = self.getboolean(sec, optname)
 				elif opttype == "int":
 					v = self.getint(sec, optname)
 				else:
-					v = self.get(sec, optname)
-				if not pOptions is None and optname in pOptions:
-					continue
+					v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v
 			except NoSectionError as e:
 				if shouldExist:
@@ -289,6 +303,12 @@ class DefinitionInitConfigReader(ConfigReader):
 		return SafeConfigParserWithIncludes.read(self._cfg, self._file)
 	
 	def getOptions(self, pOpts):
+		# overwrite static definition options with init values, supplied as
+		# direct parameters from jail-config via action[xtra1="...", xtra2=...]:
+		if self._initOpts:
+			if not pOpts:
+				pOpts = dict()
+			pOpts = _merge_dicts(pOpts, self._initOpts)
 		self._opts = ConfigReader.getOptions(
 			self, "Definition", self._configOpts, pOpts)
 		
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index 8b30f914..5e6b2b74 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -27,7 +27,7 @@ __license__ = "GPL"
 import os
 import shlex
 
-from .configreader import DefinitionInitConfigReader
+from .configreader import DefinitionInitConfigReader, _merge_dicts
 from ..server.action import CommandAction
 from ..helpers import getLogger
 
@@ -50,7 +50,9 @@ class FilterReader(DefinitionInitConfigReader):
 		return self.__file
 
 	def getCombined(self):
-		combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items()))
+		combinedopts = self._opts
+		if self._initOpts:
+			combinedopts = _merge_dicts(self._opts, self._initOpts)
 		if not len(combinedopts):
 			return {}
 		opts = CommandAction.substituteRecursiveTags(combinedopts)

From 65abc639cc7838beaf1f587eae74e98afc4a1473 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 25 Nov 2016 16:56:46 +0100
Subject: [PATCH 254/623] allow newline in extra init-parameters of
 action/filter (or interpolation of it), e. g. action[...,
 logpath="%(logpath)s"]

---
 fail2ban/client/jailreader.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index b63df5f1..d01064b2 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -43,13 +43,13 @@ logSys = getLogger(__name__)
 class JailReader(ConfigReader):
 	
 	# regex, to extract list of options:
-	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
+	optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL)
 	# regex, to iterate over single option in option list, syntax:
 	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
 	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
 	# `action = act[p1=...][p2=...]`
 	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)')
+		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
 	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)

From a2af19c9f0c7ad42e733693f1cbf3d247cded582 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 25 Nov 2016 19:06:34 +0100
Subject: [PATCH 255/623] fixed several actions, that could not work with jails
 using multiple logpath; additionally repaired execution in default shell (bad
 substitution by `${x//...}` executing in `/bin/sh`); added helper
 "action.d/helpers-common.conf", and `_grep_logs` part-command for actions
 needed grep logs from multiple log-files test cases: executing of some
 complex actions covered

---
 config/action.d/complain.conf             | 20 ++++-
 config/action.d/helpers-common.conf       | 13 +++
 config/action.d/mail-whois-lines.conf     | 29 ++++---
 config/action.d/sendmail-geoip-lines.conf | 13 +--
 config/action.d/sendmail-whois-lines.conf | 13 +--
 fail2ban/tests/files/testcase01a.log      |  4 +
 fail2ban/tests/servertestcase.py          | 96 +++++++++++++++++++++--
 7 files changed, 160 insertions(+), 28 deletions(-)
 create mode 100644 config/action.d/helpers-common.conf
 create mode 100644 fail2ban/tests/files/testcase01a.log

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index 9247803e..e4ceb35f 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -28,6 +28,10 @@
 #
 
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -54,10 +58,16 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
-	    IP=<ip>
+actionban = oifs=${IFS}; 
+              IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
+              IFS=,; ADDRESSES=$(echo $ADDRESSES)
+            IFS=${oifs}
+            IP=<ip>
             if [ ! -z "$ADDRESSES" ]; then
-                (printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])<ip>([^0-9]|$)' <logpath>) | <mailcmd> "Abuse from <ip>" <mailargs> ${ADDRESSES//,/\" \"}
+                ( printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; 
+                  printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
+                  %(_grep_logs)s;
+                ) | <mailcmd> "Abuse from <ip>" <mailargs> $ADDRESSES
             fi
 
 # Option:  actionunban
@@ -92,3 +102,7 @@ mailcmd = mail -s
 #
 mailargs =
 
+# Number of log lines to include in the email
+#
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
new file mode 100644
index 00000000..7fa8e9e4
--- /dev/null
+++ b/config/action.d/helpers-common.conf
@@ -0,0 +1,13 @@
+[DEFAULT]
+
+# Usage:
+#   _grep_logs_args = 'test'
+#   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
+#
+_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
+_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
+
+[Init]
+greplimit = tail -n <grepmax>
+grepmax = 1000
+grepopts = -m <grepmax>
\ No newline at end of file
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index 6e39c605..cbd970c9 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -17,7 +18,7 @@ before = mail-whois-common.conf
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -26,7 +27,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -40,15 +41,18 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+
+_ban_mail_content = ( printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
-            Here is more information about <ip> :\n
-            `%(_whois_command)s`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Here is more information about <ip> :\n"
+            %(_whois_command)s;
+            printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+            Fail2Ban" )
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -60,6 +64,12 @@ actionunban =
 
 [Init]
 
+# Option:  mailcmd
+# Notes.:  Your system mail command. Is passed 2 args: subject and recipient
+# Values:  CMD
+#
+mailcmd = mail -s
+
 # Default name of the chain
 #
 name = default
@@ -74,4 +84,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index 2232642c..a5616e9f 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -19,7 +20,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -33,10 +34,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "<ip>" | cut -d':' -f2-`
             AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "<ip>" | cut -d':' -f2-`
             hostname: `host -t A <ip> 2>&1`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Lines containing failures of <ip>\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -50,4 +52,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index 4156c947..e1c85928 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -25,10 +26,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
             `/usr/bin/whois <ip> || echo missing whois program`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Lines containing failures of <ip>\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -42,4 +44,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/fail2ban/tests/files/testcase01a.log b/fail2ban/tests/files/testcase01a.log
new file mode 100644
index 00000000..203f0517
--- /dev/null
+++ b/fail2ban/tests/files/testcase01a.log
@@ -0,0 +1,4 @@
+Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:02 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:03 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:04 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 56c85e94..f8915b59 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -28,6 +28,7 @@ import unittest
 import time
 import tempfile
 import os
+import re
 import sys
 import platform
 
@@ -1609,31 +1610,114 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# wrap default command processor:
 					action.executeCmd = self._executeCmd
 					# test start :
-					logSys.debug('# === start ==='); self.pruneLog()
+					self.pruneLog('# === start ===')
 					action.start()
 					self.assertLogged(*tests['start'], all=True)
 					# test ban ip4 :
-					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+					self.pruneLog('# === ban-ipv4 ===')
 					action.ban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test unban ip4 :
-					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+					self.pruneLog('# === unban ipv4 ===')
 					action.unban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test ban ip6 :
-					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+					self.pruneLog('# === ban ipv6 ===')
 					action.ban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test unban ip6 :
-					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+					self.pruneLog('# === unban ipv6 ===')
 					action.unban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test stop :
-					logSys.debug('# === stop ==='); self.pruneLog()
+					self.pruneLog('# === stop ===')
 					action.stop()
 					self.assertLogged(*tests['stop'], all=True)
 
+		def _executeMailCmd(self, realCmd, timeout=60):
+			# replace pipe to mail with pipe to cat:
+			realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)',
+				r' echo mail \1 ) | cat', realCmd)
+			# replace abuse retrieving (possible no-network):
+			realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+',
+				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
+			# execute action:
+			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
+
+		def testComplexMailActionMultiLog(self):
+			testJailsActions = (
+				# mail-whois-lines --
+				('j-mail-whois-lines', 
+					'mail-whois-lines['
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s", ' +
+						# 2 logs to test grep from multiple logs:
+					  'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' +
+				    '         ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", '
+					  '_whois_command="echo \'-- information about <ip> --\'"'
+					  ']',
+				{
+					'ip4-ban': (
+						'The IP 87.142.124.10 has just been banned by Fail2Ban after',
+						'100 attempts against j-mail-whois-lines.',
+						'Here is more information about 87.142.124.10 :',
+						'-- information about 87.142.124.10 --',
+						'Lines containing failures of 87.142.124.10 (max 2)',
+						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
+						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
+					),
+				}),
+				# complain --
+				('j-complain-abuse', 
+					'complain['
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' +
+						# 2 logs to test grep from multiple logs:
+					  'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' +
+				    '         ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", '
+					  ']',
+				{
+					'ip4-ban': (
+						'Lines containing failures of 87.142.124.10 (max 2)',
+						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
+						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
+						# both abuse mails should be separated with space:
+						'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+					),
+				}),
+			)
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			for jail, act, tests in testJailsActions:
+				stream = self.getDefaultJailStream(jail, act)
+
+				# for cmd in stream:
+				# 	print(cmd)
+
+				# transmit jail to the server:
+				for cmd in stream:
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
+
+			jails = server._Server__jails
+
+			for jail, act, tests in testJailsActions:
+				# print(jail, jails[jail])
+				for a in jails[jail].actions:
+					action = jails[jail].actions[a]
+					logSys.debug('# ' + ('=' * 50))
+					logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+					logSys.debug('# ' + ('=' * 50))
+					# wrap default command processor:
+					action.executeCmd = self._executeMailCmd
+					# test ban :
+					self.pruneLog('# === ban ===')
+					action.ban({'ip': IPAddr('87.142.124.10'), 
+						'failures': 100,
+					})
+					self.assertLogged(*tests['ip4-ban'], all=True)

From ec7bb0d6c9c06dcd9a09d8bd7338d9cfc4359a0d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 25 Nov 2016 20:12:49 +0100
Subject: [PATCH 256/623] python 3x compatibility fix (positional arguments vs
 named arguments)

---
 fail2ban/client/configreader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index caab67aa..a72ca1e9 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -141,7 +141,7 @@ class ConfigReader():
 
 	def get(self, sec, opt, raw=False, vars={}):
 		if self._cfg is not None:
-			return self._cfg.get(sec, opt, raw, vars)
+			return self._cfg.get(sec, opt, raw=raw, vars=vars)
 		return None
 
 	def getOptions(self, *args, **kwargs):

From c06084d7d98ccd8a7ce28d18fc0f81402fc9752d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 28 Nov 2016 11:04:37 +0100
Subject: [PATCH 257/623] _start_params - fix: symlinks should be absolute
 paths

---
 fail2ban/tests/fail2banclienttestcase.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index b64ecb38..e6cc46cb 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -138,8 +138,8 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 			"""Filters list of 'files' to contain only directories (under dir)"""
 			return [f for f in files if isdir(pjoin(dir, f))]
 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
-		os.symlink(pjoin(STOCK_CONF_DIR, "action.d"), pjoin(cfg, "action.d"))
-		os.symlink(pjoin(STOCK_CONF_DIR, "filter.d"), pjoin(cfg, "filter.d"))
+		os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "action.d")), pjoin(cfg, "action.d"))
+		os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "filter.d")), pjoin(cfg, "filter.d"))
 		# replace fail2ban params (database with memory):
 		r = re.compile(r'^dbfile\s*=')
 		for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True):
@@ -424,7 +424,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.execSuccess(startparams, "-vvd")
 		self.assertLogged("Loading files")
 		self.assertLogged("logtarget")
-
+		
 	@with_tmpdir
 	@with_kill_srv
 	def testClientStartBackgroundInside(self, tmp):

From 5d5ab274350035cd9899ab15652f97216dd5d759 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 28 Nov 2016 13:17:36 +0100
Subject: [PATCH 258/623] small amend: removed unreachable code + coverage
 increase

---
 fail2ban/server/datetemplate.py        |  2 --
 fail2ban/tests/datedetectortestcase.py | 17 ++++++++++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index f670cbbd..1d0b014b 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -219,7 +219,6 @@ class DateEpoch(DateTemplate):
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
 			return (float(dateMatch.group(1)), dateMatch)
-		return None
 
 
 class DatePatternRegex(DateTemplate):
@@ -338,4 +337,3 @@ class DateTai64n(DateTemplate):
 			seconds_since_epoch = value[2:17]
 			# convert seconds from HEX into local time stamp
 			return (int(seconds_since_epoch, 16), dateMatch)
-		return None
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 695f099c..15cb6f2d 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -132,14 +132,15 @@ class DateDetectorTest(LogCaptureTestCase):
 			(True,  True,  "1106513999.000", None), # Regular epoch with millisec
 			(True,  True,  "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match)
 			(False, True,  "audit(1106513999.000:987)", "1106513999.000"), # SELinux
+			(True,  True,  "no date line", None), # no date in string
 		):
-			logSys.debug('== test %r', (anchored, bound, sdate))
+			if rdate is None and sdate != "no date line": rdate = sdate
+			logSys.debug('== test %r', (anchored, bound, sdate, rdate))
 			for should_match, prefix in (
-				(True,         ""),
+				(rdate is not None, ""),
 				(not anchored, "bogus-prefix "),
 				(False,        "word-boundary")
 			):
-				if rdate is None: rdate = sdate
 				log = prefix + sdate + "[sshd] error: PAM: Authentication failure"
 				# if not allowed boundary test:
 				if not bound and prefix == "word-boundary": continue
@@ -283,6 +284,16 @@ class DateDetectorTest(LogCaptureTestCase):
 		finally:
 			datedetector.logLevel = self.__old_eff_level
 
+	def testWrongTemplate(self):
+		t = DatePatternRegex('(%ExY%Exm%Exd')
+		# lazy compiling used, so try match:
+		self.assertRaises(Exception, t.matchDate, '(20050101')
+		self.assertLogged("Compile %r failed" % t.name)
+		# abstract:
+		t = DateTemplate()
+		self.assertRaises(Exception, t.getDate, 'no date line')
+
+
 iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
 
 class CustomDateFormatsTest(unittest.TestCase):

From 39c343bd0699ae98d49a98aeeefaa4a9ecc6347a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 28 Nov 2016 15:18:31 +0100
Subject: [PATCH 259/623] better reorder templates handling, code coverage
 increase (a small part of _reorderTemplate was not covered at all)

---
 fail2ban/server/datedetector.py        | 52 +++++++++++++-------------
 fail2ban/tests/datedetectortestcase.py |  4 +-
 2 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 01cb43fe..bef3be6e 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -92,7 +92,7 @@ class DateDetectorCache(object):
 		if self.__templates:
 			return self.__templates
 		with self.__lock:
-			if self.__templates:
+			if self.__templates: # pragma: no cover - race-condition + multi-threaded environment only
 				return self.__templates
 			self._addDefaultTemplate()
 			return self.__templates
@@ -324,7 +324,7 @@ class DateDetector(object):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
 			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
-				if logSys.getEffectiveLevel() <= logLevel-1:
+				if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug
 					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
 					match = template.matchDate(line)
 					ignoreBySearch = i
@@ -452,7 +452,7 @@ class DateDetector(object):
 			try:
 				date = template.getDate(line, timeMatch[0])
 				if date is not None:
-					if logSys.getEffectiveLevel() <= logLevel:
+					if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
 						logSys.log(logLevel, "  got time %f for %r using template %s",
 							date[0], date[1].group(1), template.name)
 					return date
@@ -478,29 +478,31 @@ class DateDetector(object):
 			weight = ddtempl.weight
 			## try to move faster (first if unused available, or half of part to current template position):
 			pos = self.__firstUnused if self.__firstUnused < num else num // 2
-			pweight = templates[pos].weight
 			## don't move too often (multiline logs resp. log's with different date patterns),
 			## if template not used too long, replace it also :
-			if logSys.getEffectiveLevel() <= logLevel:
-				logSys.log(logLevel, "  -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r",
-					num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
-			if not pweight or weight > pweight or templates[pos].lastUsed < untime:
-				## if not larger (and target position recently used) - move slow (exact 1 position):
-				if weight <= pweight and templates[pos].lastUsed > untime:
-					pos = num-1
-					## if still smaller and template at position used, don't move:
-					pweight = templates[pos].weight
-					if logSys.getEffectiveLevel() <= logLevel:
-						logSys.log(logLevel, "  -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r",
-							num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
-					if weight < pweight and templates[pos].lastUsed > untime:
-						return
-				del templates[num]
-				templates[pos:0] = [ddtempl]
-				## correct first unused:
-				while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits:
-					self.__firstUnused += 1
+			def _moveable():
+				pweight = templates[pos].weight
 				if logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "  -> moved template #%02i -> #%02i", num, pos)
-				return pos
+					logSys.log(logLevel, "  -> compare template #%02i & #%02i, weight %.3f > %.3f, hits %r > %r",
+						num, pos, weight, pweight, ddtempl.hits, templates[pos].hits)
+				return weight > pweight or untime > templates[pos].lastUsed
+			##
+			## if not moveable (smaller weight or target position recently used):
+			if not _moveable():
+				## try to move slow (exact 1 position):
+				if pos == num-1:
+					return num				
+				pos = num-1
+				## if still smaller and template at position used, don't move:
+				if not _moveable():
+					return num				
+			## move:
+			del templates[num]
+			templates[pos:0] = [ddtempl]
+			## correct first unused:
+			while self.__firstUnused < len(templates) and templates[self.__firstUnused].hits:
+				self.__firstUnused += 1
+			if logSys.getEffectiveLevel() <= logLevel:
+				logSys.log(logLevel, "  -> moved template #%02i -> #%02i", num, pos)
+			return pos
 		return num
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 15cb6f2d..5b32a7e9 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -250,12 +250,12 @@ class DateDetectorTest(LogCaptureTestCase):
 			("030324  0:04:00",            "server mysqld[1000]: 030324  0:04:00 [Warning] Access denied ..."
 																				" foreign-input just some free text 2003-03-07 17:05:01 test", 10),
 			# distance collision detection (first date should be found):
-			("Sep 16 21:30:26",            "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 10),
+			("Sep 16 21:30:26",            "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 15),
 			# just to test sorting:
 			("2005-10-07 06:09:42",        "server mysqld[5906]: 2005-10-07 06:09:42 5907 [Warning] Access denied", 20),
 			("2005-10-08T15:26:18.237955", "server mysqld[5906]: 2005-10-08T15:26:18.237955 6 [Note] Access denied", 20),
 			# date format changed again:
-			("051009 10:05:30",            "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 20),
+			("051009 10:05:30",            "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 50),
 		):
 			logSys.debug('== test: %r', (debit, line, cnt))
 			for i in range(cnt):

From 8018796b45b91e5cc8894978077b45234e299e6b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 28 Nov 2016 17:17:48 +0100
Subject: [PATCH 260/623] wrong indentation (important code-piece in if
 log-level only)

---
 fail2ban/server/datedetector.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index bef3be6e..cccbf71a 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -326,8 +326,8 @@ class DateDetector(object):
 			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 				if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug
 					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
-					match = template.matchDate(line)
-					ignoreBySearch = i
+				match = template.matchDate(line)
+				ignoreBySearch = i
 			else:
 				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
 				if logSys.getEffectiveLevel() <= logLevel-1:
@@ -357,7 +357,6 @@ class DateDetector(object):
 				logSys.log(logLevel, "  ** last pattern not found - pattern change, search ...")
 		# search template and better match:
 		if not match:
-			self.__lastTemplIdx = 0x7fffffff
 			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
 			found = None, 0x7fffffff, 0x7fffffff, -1
 			i = 0
@@ -414,7 +413,7 @@ class DateDetector(object):
 			self.__lastPos = distance, line[distance-1:distance]
 			self.__lastEndPos = endpos, line[endpos:endpos+1]
 			# if not first - try to reorder current template (bubble up), they will be not sorted anymore:
-			if i:
+			if i and i != self.__lastTemplIdx:
 				i = self._reorderTemplate(i)
 			self.__lastTemplIdx = i
 			# return tuple with match and template reference used for parsing:

From feae7370ced6eabb8a542f4c7d7c3bba53c93aa7 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 28 Nov 2016 23:19:24 +0100
Subject: [PATCH 261/623] Update THANKS

---
 THANKS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/THANKS b/THANKS
index c8a019a6..8f746f29 100644
--- a/THANKS
+++ b/THANKS
@@ -110,6 +110,7 @@ SATO Kentaro
 Sean DuBois
 Sebastian Arcus
 Serg G. Brester
+Sergey Safarov
 Sireyessire
 silviogarbes
 Stefan Tatschner

From 6e18508a07f69fe47f30a1ec7789ba59e23a8101 Mon Sep 17 00:00:00 2001
From: roedie <github@roedie.nl>
Date: Sun, 11 Dec 2016 20:44:54 +0100
Subject: [PATCH 262/623] Add shorewall IPv6 support Small patch which allow
 fail2ban to use shorewall for IPv6 bans.

---
 config/action.d/shorewall.conf | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/config/action.d/shorewall.conf b/config/action.d/shorewall.conf
index f5f2c775..b5532c8a 100644
--- a/config/action.d/shorewall.conf
+++ b/config/action.d/shorewall.conf
@@ -34,6 +34,8 @@ actionstop =
 #
 actioncheck = 
 
+[Init]
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -50,10 +52,33 @@ actionban = shorewall <blocktype> <ip>
 #
 actionunban = shorewall allow <ip>
 
-[Init]
-
 # Option:  blocktype
 # Note:    This is what the action does with rules.
 #          See man page of shorewall for options that include drop, logdrop, reject, or logreject
 # Values:  STRING
 blocktype = reject
+
+[Init?family=inet6]
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = shorewall6 <blocktype> <ip>
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionunban = shorewall6 allow <ip>
+
+# Option:  blocktype
+# Note:    This is what the action does with rules.
+#          See man page of shorewall6 for options that include drop, logdrop, reject, or logreject
+# Values:  STRING
+blocktype = reject
+

From 3adc16d266627eb3ed27db172845786f5440a635 Mon Sep 17 00:00:00 2001
From: roedie <github@roedie.nl>
Date: Mon, 12 Dec 2016 20:53:58 +0100
Subject: [PATCH 263/623] Shorewall IPv6 suggested changes. Change files as
 suggested by sebres.

---
 config/action.d/shorewall.conf | 39 ++++++++++++----------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/config/action.d/shorewall.conf b/config/action.d/shorewall.conf
index b5532c8a..282b95af 100644
--- a/config/action.d/shorewall.conf
+++ b/config/action.d/shorewall.conf
@@ -34,15 +34,13 @@ actionstop =
 #
 actioncheck = 
 
-[Init]
-
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = shorewall <blocktype> <ip>
+actionban = shorewall<family> <blocktype> <ip>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -50,7 +48,15 @@ actionban = shorewall <blocktype> <ip>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = shorewall allow <ip>
+actionunban = shorewall<family> allow <ip>
+
+
+[Init]
+
+# Option:  family
+# Note:    Control which version of command is executed
+# Values:  Empty or 6 in case of IPv6
+family = 
 
 # Option:  blocktype
 # Note:    This is what the action does with rules.
@@ -60,25 +66,8 @@ blocktype = reject
 
 [Init?family=inet6]
 
-# Option:  actionban
-# Notes.:  command executed when banning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    See jail.conf(5) man page
-# Values:  CMD
-#
-actionban = shorewall6 <blocktype> <ip>
-
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    See jail.conf(5) man page
-# Values:  CMD
-#
-actionunban = shorewall6 allow <ip>
-
-# Option:  blocktype
-# Note:    This is what the action does with rules.
-#          See man page of shorewall6 for options that include drop, logdrop, reject, or logreject
-# Values:  STRING
-blocktype = reject
+# Option:  family
+# Note:    Control which version of command is executed
+# Values:  Empty or 6 in case of IPv6
+family = 6
 

From cc311b56f3add83e38d444969c2ddab618f6bf36 Mon Sep 17 00:00:00 2001
From: benrubson <ben.rubson@gmail.com>
Date: Fri, 23 Dec 2016 22:57:24 +0100
Subject: [PATCH 264/623] Apache URIs can contain spaces

---
 config/filter.d/apache-auth.conf | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf
index 8a63858d..36e5aed7 100644
--- a/config/filter.d/apache-auth.conf
+++ b/config/filter.d/apache-auth.conf
@@ -10,18 +10,18 @@ before = apache-common.conf
 [Definition]
 
 
-failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$
-            ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$
-            ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*?: password mismatch: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01631: )?user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$
+failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration
+            ^%(_apache_error_client)s (AH01617: )?user \S* authentication failure
+            ^%(_apache_error_client)s (AH01618: )?user \S* not found
+            ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme
+            ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S* to access .* failed
+            ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user \S*: password mismatch
+            ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `\S*' in realm `.+' (not found|denied by provider)
+            ^%(_apache_error_client)s (AH01631: )?user \S*: authorization failure
             ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+(, referer: \S+)?\s*$
             ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01793: )?invalid qop `.*?' received: \S*(, referer: \S+)?\s*$
+            ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `
+            ^%(_apache_error_client)s (AH01793: )?invalid qop `
             ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$
 
 ignoreregex = 
@@ -43,6 +43,8 @@ ignoreregex =
 # all of these expressions. Lots of submodules like mod_authz_* return back to mod_authz_core
 # to return the actual failure.
 #
+# Note that URI can contain spaces.
+#
 # See also: http://wiki.apache.org/httpd/ListOfErrors
 # Expressions that don't have tests and aren't common.
 # more be added with  https://issues.apache.org/bugzilla/show_bug.cgi?id=55284 

From 9fbf713d7dc662ca6707682a676f4964b15e61ac Mon Sep 17 00:00:00 2001
From: Hank Leininger <hlein@korelogic.com>
Date: Wed, 28 Dec 2016 11:14:32 -0500
Subject: [PATCH 265/623] Wait up to 30 seconds on "stop" to avoid errors.

The fail2ban server can take several seconds to shut down.  This can
make Gentoo's start-stop-service time out and decide that stopping has
failed, even if it actually succeeds a few seconds later.

The default timeout for start-stop-service if --retry is not specified
appears to be 5 seconds.  Increase that to 30 seconds to be sure that if
fail2ban-server is going to be able to stop, it has time to do so.
---
 files/gentoo-initd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/gentoo-initd b/files/gentoo-initd
index c5b5b702..c4d8675c 100755
--- a/files/gentoo-initd
+++ b/files/gentoo-initd
@@ -41,7 +41,7 @@ start() {
 
 stop() {
 	ebegin "Stopping fail2ban"
-	start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid \
+	start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid --retry 30 \
 	-- ${FAIL2BAN} stop
 	eend $? "Failed to stop fail2ban"
 }

From 81c1810f10259ad25041dc0bc82aed39d7d47948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Sat, 31 Dec 2016 21:30:57 +0100
Subject: [PATCH 266/623] Introduce Cloudflare API v4

In the cloudflare action everyone is suggested to use API v4.
And I don't dare to contribute any actual change.
---
 config/action.d/cloudflare.conf | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf
index aa87163c..15b0e31e 100644
--- a/config/action.d/cloudflare.conf
+++ b/config/action.d/cloudflare.conf
@@ -40,7 +40,12 @@ actioncheck =
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
+# API v1
 actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v4
+#actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+#            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
+#            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -50,7 +55,12 @@ actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
+# API v1
 actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+# API v4
+#actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+#              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+#              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)
 
 [Init]
 

From 55e107310fd0697d583aa8971e2b1db48e49c5ea Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett.sun@gmail.com>
Date: Sat, 7 Jan 2017 14:24:54 +0200
Subject: [PATCH 267/623] Added config for AbuseIPDB, ony tested on Ubuntu
 16.04

---
 THANKS                         |  2 +
 config/action.d/abuseipdb.conf | 74 ++++++++++++++++++++++++++++++++++
 config/jail.conf               | 21 +++++++++-
 3 files changed, 95 insertions(+), 2 deletions(-)
 create mode 100644 config/action.d/abuseipdb.conf

diff --git a/THANKS b/THANKS
index 8f746f29..7537cb92 100644
--- a/THANKS
+++ b/THANKS
@@ -16,6 +16,7 @@ Alexander Koeppe (IPv6 support)
 Alexandre Perrin (kAworu)
 Amir Caspi
 Amy
+Andrew James Collett (ajcollett)
 Andrew St. Jean
 Andrey G. Grozin
 Andy Fragen
@@ -111,6 +112,7 @@ Sean DuBois
 Sebastian Arcus
 Serg G. Brester
 Sergey Safarov
+Shaun C.
 Sireyessire
 silviogarbes
 Stefan Tatschner
diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf
new file mode 100644
index 00000000..7bd3f68a
--- /dev/null
+++ b/config/action.d/abuseipdb.conf
@@ -0,0 +1,74 @@
+# Fail2ban configuration file
+#
+# Action to report IP address to abuseipdb.com
+# You must sign up to obtain an API key from abuseipdb.com.
+#
+# IMPORTANT:
+#
+# Reporting an IP of abuse is a serious complaint. Make sure that it is
+# serious. Fail2ban developers and network owners recommend you only use this
+# action for:
+#   * The recidive where the IP has been banned multiple times
+#   * Where maxretry has been set quite high, beyond the normal user typing
+#     password incorrectly.
+#   * For filters that have a low likelihood of receiving human errors
+#
+# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
+# Added to fail2ban by Andrew James Collett (ajcollett)
+
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+actionstart =
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+actionstop =
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck =
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+#
+#          ** IMPORTANT! **
+#
+#          By default, this posts directly to AbuseIPDB's API, unfortunately
+#          this results in a lot of backslashes/escapes appearing in the
+#          reports. This also may include info like your hostname.
+#          If you have your own web server with PHP available, you can
+#          use my (Shaun's) helper PHP script by commenting out the first #actionban
+#          line below, uncommenting the second one, and pointing the URL at
+#          wherever you install the helper script. For the PHP helper script, see
+#          <https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban>
+#
+#          --ciphers ecdhe_ecdsa_aes_256_sha is used to workaround a
+#          "NSS error -12286" from curl as it attempts to connect using
+#          SSLv3. See https://www.centos.org/forums/viewtopic.php?t=52732
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = curl --fail --ciphers ecdhe_ecdsa_aes_256_sha --data 'key=<abuseipdb_apikey>' --data-urlencode 'comment=<matches>' --data 'ip=<ip>' --data 'category=<abuseipdb_category>' "https://www.abuseipdb.com/report/json"
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionunban =
+
+[Init]
+# Option:  abuseipdb_apikey
+# Notes    Your API key from abuseipdb.com
+# Values:  STRING  Default: None
+abuseipdb_apikey =
diff --git a/config/jail.conf b/config/jail.conf
index b7c927e2..f5d24621 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
                 %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
-# 
+#
 # See the IMPORTANT note in action.d/blocklist_de.conf for when to
 # use this action. Create a file jail.d/blocklist_de.local containing
 # [Init]
@@ -206,6 +206,23 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag
 # Report ban via badips.com (uses action.d/badips.conf for reporting only)
 #
 action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
+# Actions to report to abuseipdb.com via API.
+# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
+# If you want cleaner reports that ensure no user data see the helper script at the above IP
+# NOTE: These reports may include sensitive Info.
+
+# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf
+action_abuseipdb_fraud     = abuseipdb[abuseipdb_category="3"]
+action_abuseipdb_ddos      = abuseipdb[abuseipdb_category="4"]
+action_abuseipdb_proxy     = abuseipdb[abuseipdb_category="9"]
+action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"]
+action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"]
+action_abuseipdb_blogspam  = abuseipdb[abuseipdb_category="12"]
+action_abuseipdb_portscan  = abuseipdb[abuseipdb_category="14"]
+action_abuseipdb_hack      = abuseipdb[abuseipdb_category="15"]
+action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"]
+action_abuseipdb_spoofing  = abuseipdb[abuseipdb_category="17"]
+action_abuseipdb_sshbrute  = abuseipdb[abuseipdb_category="18"]
 
 # Choose default action.  To change, just override value of 'action' with the
 # interpolation to the chosen action shortcut (e.g.  action_mw, action_mwl, etc) in jail.local
@@ -330,7 +347,7 @@ logpath = /opt/openhab/logs/request.log
 port    = http,https
 logpath = %(nginx_error_log)s
 
-# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` 
+# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module`
 # and define `limit_req` and `limit_req_zone` as described in nginx documentation
 # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html
 # or for example see in 'config/filter.d/nginx-limit-req.conf'

From 1c41390f7caed9a9e6712c83d80774bf7bf1c14f Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett.sun@gmail.com>
Date: Sun, 8 Jan 2017 09:26:11 +0200
Subject: [PATCH 268/623] Restructured the way the catagories work. Jail.conf
 is cleaner and abuseipdb.conf is more flexible.

---
 config/action.d/abuseipdb.conf | 19 +++++++++++++++++++
 config/jail.conf               | 28 ++++++++++++----------------
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf
index 7bd3f68a..3bcfb065 100644
--- a/config/action.d/abuseipdb.conf
+++ b/config/action.d/abuseipdb.conf
@@ -16,6 +16,23 @@
 # Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)
 
+## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call.
+# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
+# ID	Title	Description
+# 3	  Fraud Orders
+# 4	  DDoS Attack
+# 9	  Open Proxy
+# 10	Web Spam
+# 11	Email Spam
+# 14	Port Scan
+# 18	Brute-Force
+# 19	Bad Web Bot
+# 20	Exploited Host
+# 21	Web App Attack
+# 22	SSH	Secure Shell (SSH) abuse. Use this category in combination with more specific categories.
+# 23	IoT Targeted
+# See https://abuseipdb.com/categories for more descriptions
+
 [Definition]
 
 # Option:  actionstart
@@ -71,4 +88,6 @@ actionunban =
 # Option:  abuseipdb_apikey
 # Notes    Your API key from abuseipdb.com
 # Values:  STRING  Default: None
+# Register for abuseipdb [https://www.abuseipdb.com], get api key and set below.
+# You will need to set the catagory in the action call.
 abuseipdb_apikey =
diff --git a/config/jail.conf b/config/jail.conf
index f5d24621..7ba343fb 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -206,23 +206,19 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag
 # Report ban via badips.com (uses action.d/badips.conf for reporting only)
 #
 action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
-# Actions to report to abuseipdb.com via API.
-# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
-# If you want cleaner reports that ensure no user data see the helper script at the above IP
+# Report ban via abuseipdb.com.
+#
+# See action.d/abuseipdb.conf and
+# https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
+#
 # NOTE: These reports may include sensitive Info.
-
-# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf
-action_abuseipdb_fraud     = abuseipdb[abuseipdb_category="3"]
-action_abuseipdb_ddos      = abuseipdb[abuseipdb_category="4"]
-action_abuseipdb_proxy     = abuseipdb[abuseipdb_category="9"]
-action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"]
-action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"]
-action_abuseipdb_blogspam  = abuseipdb[abuseipdb_category="12"]
-action_abuseipdb_portscan  = abuseipdb[abuseipdb_category="14"]
-action_abuseipdb_hack      = abuseipdb[abuseipdb_category="15"]
-action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"]
-action_abuseipdb_spoofing  = abuseipdb[abuseipdb_category="17"]
-action_abuseipdb_sshbrute  = abuseipdb[abuseipdb_category="18"]
+# If you want cleaner reports that ensure no user data see the helper script at the above website.
+#
+# IMPORTANT: This action relies on a api_key being added to the above action conf,
+# and the appropriate catagories set.
+# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
+# See action.d/abuseipdb.conf for catagories
+action_abuseipdb = abuseipdb
 
 # Choose default action.  To change, just override value of 'action' with the
 # interpolation to the chosen action shortcut (e.g.  action_mw, action_mwl, etc) in jail.local

From b35391e768c4a476cc176959bad1571ccfd40708 Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett@users.noreply.github.com>
Date: Sun, 8 Jan 2017 09:30:00 +0200
Subject: [PATCH 269/623] Update jail.conf

Fixing spacing
---
 config/jail.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 7ba343fb..09675904 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
                 %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
-#
+# 
 # See the IMPORTANT note in action.d/blocklist_de.conf for when to
 # use this action. Create a file jail.d/blocklist_de.local containing
 # [Init]
@@ -330,7 +330,7 @@ port    = http,https
 logpath = %(apache_error_log)s
 maxretry = 1
 
-
+ 
 [openhab-auth]
 
 filter = openhab

From 10d61e077942c25ac5a026a6c6da2b3d2bb6ee3f Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett.sun@gmail.com>
Date: Sun, 8 Jan 2017 09:39:12 +0200
Subject: [PATCH 270/623] Fixed the spaces again

---
 config/jail.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index 09675904..31f0c2be 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -330,7 +330,7 @@ port    = http,https
 logpath = %(apache_error_log)s
 maxretry = 1
 
- 
+
 [openhab-auth]
 
 filter = openhab

From 3991f51f30f10a1086f0999089aa25c8fedb653d Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett@users.noreply.github.com>
Date: Sun, 8 Jan 2017 09:45:35 +0200
Subject: [PATCH 271/623] Update jail.conf

Sigh, added a space back that I somehow missed in Vim, despite it being a rebase...
---
 config/jail.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index 31f0c2be..560136a6 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -343,7 +343,7 @@ logpath = /opt/openhab/logs/request.log
 port    = http,https
 logpath = %(nginx_error_log)s
 
-# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module`
+# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` 
 # and define `limit_req` and `limit_req_zone` as described in nginx documentation
 # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html
 # or for example see in 'config/filter.d/nginx-limit-req.conf'

From 18d09b6d8ef6c72b83fd99f81832ffe2e7c2ce8d Mon Sep 17 00:00:00 2001
From: Andrew James Collett <ajcollett.sun@gmail.com>
Date: Sun, 8 Jan 2017 09:50:58 +0200
Subject: [PATCH 272/623] Updated changelog.

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7964a9ba..5f5496c7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -83,6 +83,10 @@ TODO: implementing of options resp. other tasks from PR #1346
     if configuration is clean (fails by wrong configured jails if option `-t` specified)
 * New command action parameter `actionrepair` - command executed in order to restore
   sane environment in error case of `actioncheck`.
+* Reporting via abuseipdb.com:
+  - Bans can now be reported to abuseipdb
+  - Catagories must be set in the config
+  - Relevant log lines included in report
 
 ### Enhancements
 * Huge increasing of fail2ban performance and especially test-cases performance (see gh-1109)

From fb27d9ce83191ecf2887b84453f7ea0dae9e4c8f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 Jan 2017 10:59:53 +0100
Subject: [PATCH 273/623] fail2ban-regex: fix for systemd-journal (see gh-1657)

---
 fail2ban/client/fail2banregex.py | 31 ++++++++++++++-----------------
 fail2ban/server/filtersystemd.py |  8 ++++++++
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index ff55c417..d111e09c 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -41,12 +41,12 @@ from optparse import OptionParser, Option
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 
 try: # pragma: no cover
-	from systemd import journal
 	from ..server.filtersystemd import FilterSystemd
 except ImportError:
-	journal = None
+	FilterSystemd = None
 
 from ..version import version
+from .jailreader import JailReader
 from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import RegexException
@@ -80,7 +80,7 @@ def pprint_list(l, header=None):
 		s = ''
 	output( s + "|  " + "\n|  ".join(l) + '\n`-' )
 
-def journal_lines_gen(myjournal): # pragma: no cover
+def journal_lines_gen(flt, myjournal): # pragma: no cover
 	while True:
 		try:
 			entry = myjournal.get_next()
@@ -88,7 +88,7 @@ def journal_lines_gen(myjournal): # pragma: no cover
 			continue
 		if not entry:
 			break
-		yield FilterSystemd.formatJournalEntry(entry)
+		yield flt.formatJournalEntry(entry)
 
 def get_opt_parser():
 	# use module docstring for help output
@@ -524,25 +524,22 @@ class Fail2banRegex(object):
 			except IOError as e:
 				output( e )
 				return False
-		elif cmd_log == "systemd-journal": # pragma: no cover
-			if not journal:
+		elif cmd_log.startswith("systemd-journal"): # pragma: no cover
+			if not FilterSystemd:
 				output( "Error: systemd library not found. Exiting..." )
 				return False
-			myjournal = journal.Reader(converters={'__CURSOR': lambda x: x})
+			output( "Use         systemd journal" )
+			output( "Use         encoding : %s" % self._encoding )
+			backend, beArgs = JailReader.extractOptions(cmd_log)
+			flt = FilterSystemd(None, **beArgs)
+			flt.setLogEncoding(self._encoding)
+			myjournal = flt.getJournalReader()
 			journalmatch = self._journalmatch
 			self.setDatePattern(None)
 			if journalmatch:
-				try:
-					for element in journalmatch:
-						if element == "+":
-							myjournal.add_disjunction()
-						else:
-							myjournal.add_match(element)
-				except ValueError:
-					output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) )
-					return False
+				flt.addJournalMatch(journalmatch)
 			output( "Use    journal match : %s" % " ".join(journalmatch) )
-			test_lines = journal_lines_gen(myjournal)
+			test_lines = journal_lines_gen(flt, myjournal)
 		else:
 			output( "Use      single line : %s" % shortstr(cmd_log) )
 			test_lines = [ cmd_log ]
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 0d720a5e..d858142b 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -178,6 +178,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def getJournalMatch(self):
 		return self.__matches
 
+	##
+	# Get journal reader
+	#
+	# @return journal reader
+
+	def getJournalReader(self):
+		return self.__journal
+
 	##
 	# Format journal log entry into syslog style
 	#

From bf872213bdad3228510a977740eaac3aac2d612a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 10 Jan 2017 13:48:17 +0100
Subject: [PATCH 274/623] amend for 7019640eb3d1f901e52b4c00cb6d029e82fdda17
 (fix-gh-1658): sshd test-cases extended with IPv6 to cover this fix

---
 fail2ban/tests/files/logs/sshd | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index 15ec4274..0879a2da 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -9,8 +9,14 @@ May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-i
 #2
 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" }
 Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2
+# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" }
+Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 50273 ssh2
 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" }
 Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345
+# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" }
+Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 12345
+# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" }
+Feb 25 14:34:11 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1
 
 #3
 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
@@ -118,6 +124,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po
 
 # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" }
 Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4
+# failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" }
+Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4
 
 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" }
 Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2
@@ -128,6 +136,8 @@ Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from
 
 # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." }
 Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
+# failJSON: { "time": "2005-07-05T18:22:45", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Failed publickey for ..." }
+Jul 05 18:22:45 mercury sshd[4670]: Failed publickey for graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
 
 # failJSON: { "match": false }
 Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth]

From 6f190b6e611cfee8796f6720e8df17c73e0e63ee Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 11 Jan 2017 19:04:33 +0100
Subject: [PATCH 275/623] readme.md: added IPv6 launch logo for 0.10th branch

Closes gh-1647
---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index be0be636..ee654acb 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,9 @@ Though Fail2Ban is able to reduce the rate of incorrect authentications
 attempts, it cannot eliminate the risk that weak authentication presents.
 Configure services to use only two factor or public/private authentication
 mechanisms if you really want to protect services.
+     
+<img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of the IPv6 addresses.
+------|------
 
 This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs
 are available in fail2ban(1) manpage and on the website http://www.fail2ban.org

From 4a65e069e1437d165b3265d6caba623e02327462 Mon Sep 17 00:00:00 2001
From: oliverdorn <developer@veriosmart.de>
Date: Thu, 12 Jan 2017 22:07:46 +0100
Subject: [PATCH 276/623] Solution for issue #1665

Solves the issue of authentic GoogleBots being banned by apache-fakegooglebots.
---
 files/debian-initd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/debian-initd b/files/debian-initd
index 0d2bed9a..d6660215 100755
--- a/files/debian-initd
+++ b/files/debian-initd
@@ -22,7 +22,7 @@
 #  rename this file: (sudo) mv /etc/init.d/fail2ban.init /etc/init.d/fail2ban
 #  same with the logrotate file: (sudo) mv /etc/logrotate.d/fail2ban.logrotate /etc/logrotate.d/fail2ban
 #
-PATH=/usr/sbin:/usr/bin:/sbin:/bin
+PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin
 DESC="authentication failure monitor"
 NAME=fail2ban
 

From ee3c787cc68f6d043c8702c8f53a566277e2ee02 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 Jan 2017 19:06:17 +0100
Subject: [PATCH 277/623] Recognize restored (from database) tickets after
 restart (tell action restored state of the ticket); Prevent executing of
 several actions (e.g. mail, send-mail etc) on restart (bans were already
 notified). Test cases extended (smtp and by restart in ServerReloadTest).
 Closes gh-1141 Closes gh-921

---
 config/action.d/complain.conf                 |  3 +-
 config/action.d/dshield.conf                  | 10 ++++-
 config/action.d/helpers-common.conf           |  3 ++
 config/action.d/mail-buffered.conf            |  8 +++-
 config/action.d/mail-whois-lines.conf         |  4 +-
 config/action.d/mail-whois.conf               |  4 +-
 config/action.d/mail.conf                     |  7 +++-
 config/action.d/sendmail-buffered.conf        |  4 +-
 config/action.d/sendmail-geoip-lines.conf     |  3 +-
 .../sendmail-whois-ipjailmatches.conf         |  4 +-
 config/action.d/sendmail-whois-ipmatches.conf |  4 +-
 config/action.d/sendmail-whois-lines.conf     |  3 +-
 config/action.d/sendmail-whois-matches.conf   |  4 +-
 config/action.d/sendmail-whois.conf           |  4 +-
 config/action.d/sendmail.conf                 |  4 +-
 config/action.d/smtp.py                       |  2 +
 config/action.d/xarf-login-attack.conf        |  7 +++-
 fail2ban/server/action.py                     |  4 +-
 fail2ban/server/actions.py                    |  4 ++
 fail2ban/tests/action_d/test_smtp.py          | 15 ++++++-
 fail2ban/tests/fail2banclienttestcase.py      | 42 +++++++++++++++----
 21 files changed, 114 insertions(+), 29 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index e4ceb35f..d0156a44 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -58,7 +58,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; 
+actionban = %(_bypass_if_restored)s
+            oifs=${IFS}; 
               IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
               IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf
index a0041986..35eaa3be 100644
--- a/config/action.d/dshield.conf
+++ b/config/action.d/dshield.conf
@@ -26,6 +26,10 @@
 # configure how often the buffer is flushed).
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -64,7 +68,8 @@ actioncheck =
 # few seconds out, are incorrect. See
 # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047
 #
-actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
+actionban = %(_bypass_if_restored)s
+            TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
             DATETIME="`perl -e '@t=localtime(<time>);printf "%%4d-%%02d-%%02d %%02d:%%02d:%%02d",1900+$t[5],$t[4]+1,$t[3],$t[2],$t[1],$t[0]'` $TZONE"
 	    PROTOCOL=`awk '{IGNORECASE=1;if($1=="<protocol>"){print $2;exit}}' /etc/protocols`
 	    if [ -z "$PROTOCOL" ]; then PROTOCOL=<protocol>; fi
@@ -91,7 +96,8 @@ actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = if [ -f <tmpfile>.first ]; then
+actionunban = %(_bypass_if_restored)s
+              if [ -f <tmpfile>.first ]; then
                   NOW=`date +%%s`
                   LOGAGE=$(($NOW - `cat <tmpfile>.first`))
                   if [ $LOGAGE -gt <maxbufferage> ]; then
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index 7fa8e9e4..b04c7f7a 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -7,6 +7,9 @@
 _grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
 _grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
 
+# Used for actions, that should not by executed if ticket was restored:
+_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
+
 [Init]
 greplimit = tail -n <grepmax>
 grepmax = 1000
diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf
index 914d4a5a..c1b3f565 100644
--- a/config/action.d/mail-buffered.conf
+++ b/config/action.d/mail-buffered.conf
@@ -4,6 +4,11 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
+
 [Definition]
 
 # Option:  actionstart
@@ -45,7 +50,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = %(_bypass_if_restored)s
+            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Hi,\n
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index cbd970c9..c471a44a 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -52,7 +52,9 @@ _ban_mail_content = ( printf %%b "Hi,\n
             printf %%b "\n
             Regards,\n
             Fail2Ban" )
-actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+
+actionban = %(_bypass_if_restored)s
+            %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf
index 018c327d..045da5ec 100644
--- a/config/action.d/mail-whois.conf
+++ b/config/action.d/mail-whois.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -40,7 +41,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+actionban = %(_bypass_if_restored)s
+            printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf
index 7bf51a1d..b76c7fd3 100644
--- a/config/action.d/mail.conf
+++ b/config/action.d/mail.conf
@@ -4,6 +4,10 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -36,7 +40,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+actionban = %(_bypass_if_restored)s
+            printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf
index 80eb20a3..67b84251 100644
--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -58,7 +59,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = %(_bypass_if_restored)s
+            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index a5616e9f..a0de8f17 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -20,7 +20,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf
index 689ffe45..2a8fcbcd 100644
--- a/config/action.d/sendmail-whois-ipjailmatches.conf
+++ b/config/action.d/sendmail-whois-ipjailmatches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf
index b06e6db6..b6de1a8d 100644
--- a/config/action.d/sendmail-whois-ipmatches.conf
+++ b/config/action.d/sendmail-whois-ipmatches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index e1c85928..77972acc 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -17,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf
index 8bca5937..7122d923 100644
--- a/config/action.d/sendmail-whois-matches.conf
+++ b/config/action.d/sendmail-whois-matches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf
index 55b80bc5..82bc202e 100644
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf
index 5f5670c3..15f49766 100644
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py
index 2429cf48..65d50011 100644
--- a/config/action.d/smtp.py
+++ b/config/action.d/smtp.py
@@ -211,6 +211,8 @@ class SMTPAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
+		if aInfo.get('restored'):
+			return
 		aInfo.update(self.message_values)
 		message = "".join([
 			messages['ban']['head'],
diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 3ab73817..13a5b70c 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -30,6 +30,10 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 actionstart =
@@ -38,7 +42,8 @@ actionstop =
 
 actioncheck =
 
-actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
+actionban = %(_bypass_if_restored)s
+            oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
             IP=<ip>
             FROM=<sender>
             SERVICE=<service>
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 62aa51dc..41c58068 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -219,9 +219,9 @@ class CommandAction(ActionBase):
 			self.timeout = 60
 			## Command executed in order to initialize the system.
 			self.actionstart = ''
-			## Command executed when an IP address gets banned.
+			## Command executed when ticket gets banned.
 			self.actionban = ''
-			## Command executed when an IP address gets removed.
+			## Command executed when ticket gets removed.
 			self.actionunban = ''
 			## Command executed in order to check requirements.
 			self.actioncheck = ''
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index dd4c97c2..cc832491 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -348,6 +348,8 @@ class Actions(JailThread, Mapping):
 			aInfo["failures"] = bTicket.getAttempt()
 			aInfo["time"] = bTicket.getTime()
 			aInfo["matches"] = "\n".join(bTicket.getMatches())
+			# to bypass actions, that should not be executed for restored tickets
+			aInfo["restored"] = 1 if ticket.restored else 0
 			if self._jail.database is not None:
 				mi4ip = lambda overalljails=False, self=self, \
 					mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
@@ -449,6 +451,8 @@ class Actions(JailThread, Mapping):
 		aInfo["failures"] = ticket.getAttempt()
 		aInfo["time"] = ticket.getTime()
 		aInfo["matches"] = "".join(ticket.getMatches())
+		# to bypass actions, that should not be executed for restored tickets
+		aInfo["restored"] = 1 if ticket.restored else 0
 		if actions is None:
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index d9ad0f3a..1dba454c 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -94,16 +94,21 @@ class SMTPActionTest(unittest.TestCase):
 			"Subject: [Fail2Ban] %s: stopped" %
 				self.jail.name in self.smtpd.data)
 
-	def testBan(self):
+	def _testBan(self, restored=False):
 		aInfo = {
 			'ip': "127.0.0.2",
 			'failures': 3,
 			'matches': "Test fail 1\n",
 			'ipjailmatches': "Test fail 1\nTest Fail2\n",
 			'ipmatches': "Test fail 1\nTest Fail2\nTest Fail3\n",
-			}
+		}
+		if restored:
+			aInfo['restored'] = 1
 
 		self.action.ban(aInfo)
+		if restored: # no mail, should raises attribute error:
+			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
+			return
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		subject = "Subject: [Fail2Ban] %s: banned %s" % (
@@ -123,6 +128,12 @@ class SMTPActionTest(unittest.TestCase):
 		self.action.matches = "ipmatches"
 		self.action.ban(aInfo)
 		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
+	
+	def testBan(self):
+		self._testBan()
+
+	def testNOPByRestored(self):
+		self._testBan(restored=True)
 
 	def testOptions(self):
 		self.action.start()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e6cc46cb..5d285b76 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -756,9 +756,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				return
 			_write_file(fn, "w",
 				"[Definition]",
+				"restore = ",
 				"actionstart =  echo '[<name>] %s: ** start'" % actname, start,
 				"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
-				"actionban =    echo '[<name>] %s: ++ ban <ip>'" % actname, ban,
+				"actionban =    echo '[<name>] %s: ++ ban <ip> %%(restore)s'" % actname, ban,
 				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
 				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
 			)
@@ -777,17 +778,22 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",
-				"         test-action1[name='%(__name__)s']" if 1 in actions else "",
-				"         test-action2[name='%(__name__)s']" if 2 in actions else "",
+				"         test-action1[name='%(__name__)s']" \
+					if 1 in actions else "",
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+					if 2 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
 				"failregex = ^\s*failure (401|403) from <HOST>",
-				"            ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
+				"            ^\s*error (401|403) from <HOST>" \
+					if 2 in enabled else "",
 				"enabled = true" if 1 in enabled else "",
 				"",
 				"[test-jail2]", "backend = " + backend, "filter =", 
-				"action =",
+				"action = ",
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+					if 2 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
@@ -821,6 +827,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action1: ** start'", 
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
+		# test restored is 0:
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
+			all=True, wait=MID_WAITTIME)
 
 		# broken jail was logged (in client and server log):
 		self.assertLogged(
@@ -882,10 +892,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertNotLogged(
 			"stdout: '[test-jail1] test-action1: -- unban 192.0.2.1'")
 		
-		# don't need both actions anymore:
+		# don't need action1 anymore:
 		_write_action_cfg(actname="test-action1", allow=False)
-		_write_action_cfg(actname="test-action2", allow=False)
-		_write_jail_cfg(actions=[])
+		# leave action2 just to test restored interpolation:
+		_write_jail_cfg(actions=[2])
 		
 		# write new failures:
 		self.pruneLog("[test-phase 2b]")
@@ -913,7 +923,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Found 192.0.2.2", 
 			"[test-jail2] Ban 192.0.2.2",
 			"[test-jail2] Found 192.0.2.3", 
-			"[test-jail2] Ban 192.0.2.3", all=True)
+			"[test-jail2] Ban 192.0.2.3", 
+			all=True)
 
 		# rotate logs:
 		_write_file(test1log, "w+")
@@ -936,6 +947,19 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Restore Ban 192.0.2.4",
 			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
+		# test restored is 1:
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
+			all=True, wait=MID_WAITTIME)
+		self.assertNotLogged(
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 0'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 0'",
+			all=True)
+
+		# don't need actions anymore:
+		_write_action_cfg(actname="test-action2", allow=False)
+		_write_jail_cfg(actions=[])
 
 		# restart jail with unban all:
 		self.pruneLog("[test-phase 2d]")

From de49f0c27f3391cbb5e5eade6324365af2f7bfa5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 Jan 2017 19:33:16 +0100
Subject: [PATCH 278/623] ChangeLog entry

---
 ChangeLog | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2c202c07..8e7b5522 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -165,6 +165,28 @@ fail2ban-client set loglevel INFO
   - faster match and fewer searching of appropriate templates
     (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
   - several standard filters extended with exact prefixed or anchored date templates;
+* Added possibility to recognize restored state of the tickets (see gh-1669).
+  For the shell-based actions it is enough to add following script-piece at begin 
+  of `actionban` (or `actionunban`), to prevent execution for the restored 
+  (after restart) tickets.
+  `if [ '<restored>' = '1' ]; then exit 0; fi;`
+  Several actions extended now:
+  - complain.conf
+  - dshield.conf
+  - mail-buffered.conf
+  - mail-whois-lines.conf
+  - mail-whois.conf
+  - mail.conf
+  - sendmail-buffered.conf
+  - sendmail-geoip-lines.conf
+  - sendmail-whois-ipjailmatches.conf
+  - sendmail-whois-ipmatches.conf
+  - sendmail-whois-lines.conf
+  - sendmail-whois-matches.conf
+  - sendmail-whois.conf
+  - sendmail.conf
+  - smtp.py
+  - xarf-login-attack.conf
 * fail2ban-testcases:
   - `assertLogged` extended with parameter wait (to wait up to specified timeout,
     before we throw assert exception) + test cases rewritten using that

From 2ed2e7810d1f6c594b2d596857c80660dec8b584 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Sat, 14 Jan 2017 00:17:23 +0100
Subject: [PATCH 279/623] normalization of DefinitionInitConfigReader (action /
 filter): client-side interpolation, etc.

---
 fail2ban/client/actionreader.py        | 17 ++++++++++++++---
 fail2ban/client/configreader.py        | 23 +++++++++++++++++++++++
 fail2ban/client/filterreader.py        | 16 +++-------------
 fail2ban/client/jailreader.py          |  3 ++-
 fail2ban/server/action.py              | 15 ++++++++++-----
 fail2ban/tests/clientreadertestcase.py | 12 +++++-------
 6 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index e5025fa3..55ceda93 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -43,10 +43,15 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionrepair": ["string", None],
 		"actionban": ["string", None],
 		"actionunban": ["string", None],
+		"norestored": ["string", None],
 	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
-		self._name = initOpts.get("actname", file_)
+		actname = initOpts.get("actname")
+		if actname is None:
+			actname = file_
+			initOpts["actname"] = actname
+		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
 
@@ -64,16 +69,22 @@ class ActionReader(DefinitionInitConfigReader):
 		return self._name
 
 	def convert(self):
+		opts = self.getCombined(ignore=('timeout', 'bantime'))
+		# type-convert only after combined (otherwise boolean converting prevents substitution):
+		if opts.get('norestored'):
+			opts['norestored'] = self._convert_to_boolean(opts['norestored'])
+		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
 		multi = []
-		for opt, optval in self._opts.iteritems():
+		for opt, optval in opts.iteritems():
 			if opt in self._configOpts:
 				multi.append([opt, optval])
 		if self._initOpts:
 			for opt, optval in self._initOpts.iteritems():
-				multi.append([opt, optval])
+				if opt not in self._configOpts:
+					multi.append([opt, optval])
 		if len(multi) > 1:
 			stream.append(["multi-set", self._jailName, "action", self._name, multi])
 		elif len(multi):
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index a72ca1e9..7840cd12 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -30,6 +30,7 @@ from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
+from ..server.action import CommandAction
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -319,6 +320,28 @@ class DefinitionInitConfigReader(ConfigReader):
 					self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
+
+	def _convert_to_boolean(self, value):
+		return value.lower() in ("1", "yes", "true", "on")
+	
+	def getCombined(self, ignore=()):
+		combinedopts = self._opts
+		ignore = set(ignore).copy()
+		if self._initOpts:
+			combinedopts = _merge_dicts(self._opts, self._initOpts)
+		if not len(combinedopts):
+			return {}
+		# ignore conditional options:
+		for n in combinedopts:
+			cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+			if cond:
+				n, cond = cond.groups()
+				ignore.add(n)
+		# substiture options already specified direct:
+		opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore)
+		if not opts:
+			raise ValueError('recursive tag definitions unable to be resolved')
+		return opts
 	
 	def convert(self):
 		raise NotImplementedError
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index cdf0d8af..d89ef5ad 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -27,8 +27,7 @@ __license__ = "GPL"
 import os
 import shlex
 
-from .configreader import DefinitionInitConfigReader, _merge_dicts
-from ..server.action import CommandAction
+from .configreader import DefinitionInitConfigReader
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -52,17 +51,6 @@ class FilterReader(DefinitionInitConfigReader):
 	def getFile(self):
 		return self.__file
 
-	def getCombined(self):
-		combinedopts = self._opts
-		if self._initOpts:
-			combinedopts = _merge_dicts(self._opts, self._initOpts)
-		if not len(combinedopts):
-			return {}
-		opts = CommandAction.substituteRecursiveTags(combinedopts)
-		if not opts:
-			raise ValueError('recursive tag definitions unable to be resolved')
-		return opts
-	
 	def convert(self):
 		stream = list()
 		opts = self.getCombined()
@@ -70,6 +58,7 @@ class FilterReader(DefinitionInitConfigReader):
 			return stream
 		for opt, value in opts.iteritems():
 			if opt in ("failregex", "ignoreregex"):
+				if value is None: continue
 				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
@@ -87,6 +76,7 @@ class FilterReader(DefinitionInitConfigReader):
 				stream.append(["set", self._jailName, "datepattern", value])
 			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
+				if value is None: continue
 				for match in value.split("\n"):
 					if match == '': continue
 					stream.append(
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index b223532e..f4adadbf 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -136,7 +136,8 @@ class JailReader(ConfigReader):
 				if not filterName:
 					raise JailDefError("Invalid filter definition %r" % flt)
 				self.__filter = FilterReader(
-					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
+					filterName, self.__name, filterOpt, 
+					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				# merge options from filter as 'known/...':
 				self.__filter.getOptions(self.__opts)
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 41c58068..46a19cd1 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -365,7 +365,7 @@ class CommandAction(ActionBase):
 		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
-	def substituteRecursiveTags(cls, inptags, conditional=''):
+	def substituteRecursiveTags(cls, inptags, conditional='', ignore=()):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -387,21 +387,26 @@ class CommandAction(ActionBase):
 		# copy return tags dict to prevent modifying of inptags:
 		tags = inptags.copy()
 		t = TAG_CRE
+		ignore = set(ignore)
+		done = cls._escapedTags.copy() | ignore
 		# repeat substitution while embedded-recursive (repFlag is True)
-		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				# ignore escaped or already done:
+				# ignore escaped or already done (or in ignore list):
 				if tag in done: continue
-				value = str(tags[tag])
+				value = orgval = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
 				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
+					# don't replace tags that should be currently ignored (pre-replacement):
+					if found_tag in ignore: 
+						m = t.search(value, m.end())
+						continue
 					#logSys.log(5, 'found: %s' % found_tag)
 					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
@@ -429,7 +434,7 @@ class CommandAction(ActionBase):
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
-				if tags[tag] != value:
+				if orgval != value:
 					# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 					if t.search(value):
 						repFlag = True
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 39622cd0..f8bd3975 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -347,7 +347,7 @@ class FilterReaderTest(unittest.TestCase):
 			['set', 'testcase01', 'addjournalmatch',
 				"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
 			['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
-			['set', 'testcase01', 'maxlines', "1"], # Last for overide test
+			['set', 'testcase01', 'maxlines', 1], # Last for overide test
 		]
 		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
@@ -517,12 +517,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['add', 'brokenaction', 'auto'],
 			 ['set', 'brokenaction', 'addfailregex', '<IP>'],
 			 ['set', 'brokenaction', 'addaction', 'brokenaction'],
-			 ['set',
-			  'brokenaction',
-			  'action',
-			  'brokenaction',
-			  'actionban',
-			  'hit with big stick <ip>'],
+			 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
+				 ['actionban', 'hit with big stick <ip>'],
+				 ['actname', 'brokenaction']
+			 ]],
 			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
 			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
 			 ['start', 'emptyaction'],

From 0aa241d303fa64510252bf92988f6615baa9f8ab Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:05:45 +0100
Subject: [PATCH 280/623] Another way to recognize restored tickets - new
 option `norestored` of action introduced; Complete prevents executing of
 ban/unban operations for actions where norestored = true.

---
 fail2ban/server/actions.py               |  4 +++
 fail2ban/tests/fail2banclienttestcase.py | 32 ++++++++++++++++--------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index cc832491..5bd00c85 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -363,6 +363,8 @@ class Actions(JailThread, Mapping):
 				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
+						if ticket.restored and getattr(action, 'norestored', False):
+							continue
 						action.ban(aInfo.copy())
 					except Exception as e:
 						logSys.error(
@@ -457,6 +459,8 @@ class Actions(JailThread, Mapping):
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
 			try:
+				if ticket.restored and getattr(action, 'norestored', False):
+					continue
 				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"])
 				action.unban(aInfo.copy())
 			except Exception as e:
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 5d285b76..b8417be5 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -755,13 +755,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				os.remove(fn)
 				return
 			_write_file(fn, "w",
+				"[DEFAULT]",
+				"_exec_once = 0",
+				"",
 				"[Definition]",
+				"norestored = %(_exec_once)s",
 				"restore = ",
-				"actionstart =  echo '[<name>] %s: ** start'" % actname, start,
-				"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
-				"actionban =    echo '[<name>] %s: ++ ban <ip> %%(restore)s'" % actname, ban,
-				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
-				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
+				"actionstart =  echo '[%(name)s] %(actname)s: ** start'", start,
+				"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
+				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s'", ban,
+				"actionunban =  echo '[%(name)s] %(actname)s: -- unban <ip>'", unban,
+				"actionstop =   echo '[%(name)s] %(actname)s: __ stop'", stop,
 			)
 			if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
 				_out_file(fn)
@@ -782,6 +786,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 					if 1 in actions else "",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
 					if 2 in actions else "",
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+					if 3 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
@@ -794,6 +800,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
 					if 2 in actions else "",
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+					if 3 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
@@ -804,7 +812,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_action_cfg(actname="test-action1")
 		_write_action_cfg(actname="test-action2")
 
-		_write_jail_cfg(enabled=[1], actions=[1,2])
+		_write_jail_cfg(enabled=[1], actions=[1,2,3])
 		# append one wrong configured jail:
 		_write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", 
 			"", "filter = broken-jail-filter", "enabled = true")
@@ -827,9 +835,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action1: ** start'", 
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
-		# test restored is 0:
+		# test restored is 0 (both actions available):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
+			"stdout: '[test-jail1] test-action3: ++ ban 192.0.2.1 restored: 0'",
 			all=True, wait=MID_WAITTIME)
 
 		# broken jail was logged (in client and server log):
@@ -895,7 +904,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# don't need action1 anymore:
 		_write_action_cfg(actname="test-action1", allow=False)
 		# leave action2 just to test restored interpolation:
-		_write_jail_cfg(actions=[2])
+		_write_jail_cfg(actions=[2,3])
 		
 		# write new failures:
 		self.pruneLog("[test-phase 2b]")
@@ -947,14 +956,15 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Restore Ban 192.0.2.4",
 			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
-		# test restored is 1:
+		# test restored is 1 (only test-action2):
 		self.assertLogged(
 			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
 			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
 			all=True, wait=MID_WAITTIME)
+		# test test-action3 not executed at all (norestored check):
 		self.assertNotLogged(
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 0'",
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 0'",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.4 restored: 1'",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'",
 			all=True)
 
 		# don't need actions anymore:

From 8b82c6669e5d33fc41c72e59e7352f199e74c8bf Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:34:10 +0100
Subject: [PATCH 281/623] provide name of action to fail-message (e. g. if
 interpolation fails)

---
 fail2ban/tests/clientreadertestcase.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index f8bd3975..4a692598 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -546,7 +546,10 @@ class JailsReaderTest(LogCaptureTestCase):
 				actionName = os.path.basename(actionConfig).replace('.conf', '')
 				actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR)
 				self.assertTrue(actionReader.read())
-				actionReader.getOptions({})	  # populate _opts
+				try:
+					actionReader.getOptions({})	  # populate _opts
+				except Exception as e: # pragma: no cover
+					self.fail("action %r\n%s: %s" % (actionName, type(e).__name__, e))
 				if not actionName.endswith('-common'):
 					self.assertIn('Definition', actionReader.sections(),
 						msg="Action file %r is lacking [Definition] section" % actionConfig)

From 74a6afadd565b679f73b88a6ebf81a358c0ee155 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:40:48 +0100
Subject: [PATCH 282/623] Mail-actions switched to use new option "norestored"
 instead of checking of variable `restored` during shell execution (prevents
 executing of such actions at all).

---
 config/action.d/complain.conf                     |  6 ++++--
 config/action.d/dshield.conf                      | 13 +++++--------
 config/action.d/mail-buffered.conf                | 11 ++++-------
 config/action.d/mail-whois-lines.conf             |  6 ++++--
 config/action.d/mail-whois.conf                   |  7 ++++---
 config/action.d/mail.conf                         | 10 ++++------
 config/action.d/sendmail-buffered.conf            |  7 ++++---
 config/action.d/sendmail-geoip-lines.conf         |  6 ++++--
 config/action.d/sendmail-whois-ipjailmatches.conf |  7 ++++---
 config/action.d/sendmail-whois-ipmatches.conf     |  7 ++++---
 config/action.d/sendmail-whois-lines.conf         |  6 ++++--
 config/action.d/sendmail-whois-matches.conf       |  7 ++++---
 config/action.d/sendmail-whois.conf               |  7 ++++---
 config/action.d/sendmail.conf                     |  7 ++++---
 config/action.d/smtp.py                           |  3 +++
 config/action.d/xarf-login-attack.conf            | 10 ++++------
 16 files changed, 64 insertions(+), 56 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index d0156a44..84dbaf39 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -34,6 +34,9 @@ before = helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -58,8 +61,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            oifs=${IFS}; 
+actionban = oifs=${IFS}; 
               IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
               IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf
index 35eaa3be..4f2e09ca 100644
--- a/config/action.d/dshield.conf
+++ b/config/action.d/dshield.conf
@@ -26,12 +26,11 @@
 # configure how often the buffer is flushed).
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -68,8 +67,7 @@ actioncheck =
 # few seconds out, are incorrect. See
 # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047
 #
-actionban = %(_bypass_if_restored)s
-            TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
+actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
             DATETIME="`perl -e '@t=localtime(<time>);printf "%%4d-%%02d-%%02d %%02d:%%02d:%%02d",1900+$t[5],$t[4]+1,$t[3],$t[2],$t[1],$t[0]'` $TZONE"
 	    PROTOCOL=`awk '{IGNORECASE=1;if($1=="<protocol>"){print $2;exit}}' /etc/protocols`
 	    if [ -z "$PROTOCOL" ]; then PROTOCOL=<protocol>; fi
@@ -96,8 +94,7 @@ actionban = %(_bypass_if_restored)s
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = %(_bypass_if_restored)s
-              if [ -f <tmpfile>.first ]; then
+actionunban = if [ -f <tmpfile>.first ]; then
                   NOW=`date +%%s`
                   LOGAGE=$(($NOW - `cat <tmpfile>.first`))
                   if [ $LOGAGE -gt <maxbufferage> ]; then
diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf
index c1b3f565..e74db9cc 100644
--- a/config/action.d/mail-buffered.conf
+++ b/config/action.d/mail-buffered.conf
@@ -4,13 +4,11 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -50,8 +48,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Hi,\n
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index c471a44a..0852ba8f 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -11,6 +11,9 @@ before = mail-whois-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -53,8 +56,7 @@ _ban_mail_content = ( printf %%b "Hi,\n
             Regards,\n
             Fail2Ban" )
 
-actionban = %(_bypass_if_restored)s
-            %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf
index 045da5ec..553bfb69 100644
--- a/config/action.d/mail-whois.conf
+++ b/config/action.d/mail-whois.conf
@@ -7,10 +7,12 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -41,8 +43,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Hi,\n
+actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf
index b76c7fd3..4715ecc5 100644
--- a/config/action.d/mail.conf
+++ b/config/action.d/mail.conf
@@ -4,12 +4,11 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -40,8 +39,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Hi,\n
+actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf
index 67b84251..a91a6957 100644
--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@@ -7,10 +7,12 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -59,8 +61,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index a0de8f17..34c3aedd 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -11,6 +11,9 @@ before = sendmail-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  Command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -20,8 +23,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf
index 2a8fcbcd..5bcefe89 100644
--- a/config/action.d/sendmail-whois-ipjailmatches.conf
+++ b/config/action.d/sendmail-whois-ipjailmatches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf
index b6de1a8d..4a8edcb7 100644
--- a/config/action.d/sendmail-whois-ipmatches.conf
+++ b/config/action.d/sendmail-whois-ipmatches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index 77972acc..e3a1c974 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -11,14 +11,16 @@ before = sendmail-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf
index 7122d923..fc4ba061 100644
--- a/config/action.d/sendmail-whois-matches.conf
+++ b/config/action.d/sendmail-whois-matches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf
index 82bc202e..b8d99423 100644
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf
index 15f49766..62c94439 100644
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py
index 65d50011..aa85b259 100644
--- a/config/action.d/smtp.py
+++ b/config/action.d/smtp.py
@@ -126,6 +126,9 @@ class SMTPAction(ActionBase):
 			bantime = self._jail.actions.getBanTime,
 			)
 
+		# bypass ban/unban for restored tickets
+		self.norestored = 1
+
 	def _sendMessage(self, subject, text):
 		"""Sends message based on arguments and instance's properties.
 
diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 13a5b70c..5274cdaf 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -30,20 +30,18 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 actionstart =
 
 actionstop =
 
 actioncheck =
 
-actionban = %(_bypass_if_restored)s
-            oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
+actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
             IP=<ip>
             FROM=<sender>
             SERVICE=<service>

From f35da076df5d0c02947f5e8cf5f6092925f6e55f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:55:01 +0100
Subject: [PATCH 283/623] ChangeLog entry

---
 ChangeLog | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e7b5522..37d74955 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -166,11 +166,14 @@ fail2ban-client set loglevel INFO
     (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
   - several standard filters extended with exact prefixed or anchored date templates;
 * Added possibility to recognize restored state of the tickets (see gh-1669).
-  For the shell-based actions it is enough to add following script-piece at begin 
-  of `actionban` (or `actionunban`), to prevent execution for the restored 
-  (after restart) tickets.
+  New option `norestored` introduced, to ignore restored tickets (after restart).
+  To avoid execution of ban/unban for the restored tickets, `norestored = true`
+  could be added in definition section of action.
+  For conditional usage in the shell-based actions an interpolation `<restored>` 
+  could be used also. E. g. it is enough to add following script-piece at begin
+  of `actionban` (or `actionunban`) to prevent execution:
   `if [ '<restored>' = '1' ]; then exit 0; fi;`
-  Several actions extended now:
+  Several actions extended now using `norestored` option:
   - complain.conf
   - dshield.conf
   - mail-buffered.conf

From 901eeff53df0b3b73d607212aa9b1fd526617176 Mon Sep 17 00:00:00 2001
From: Jan Grewe <jan@faked.org>
Date: Mon, 6 Feb 2017 22:04:36 +0100
Subject: [PATCH 284/623] Make Abusix lookup compatible with Dash

---
 config/action.d/complain.conf | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index 84dbaf39..9934bd94 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -62,8 +62,9 @@ actioncheck =
 # Values:  CMD
 #
 actionban = oifs=${IFS}; 
-              IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
-              IFS=,; ADDRESSES=$(echo $ADDRESSES)
+            REV_IP=$(printf %%s ${IP}. | tac -s.)
+            ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org)
+            IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
             IP=<ip>
             if [ ! -z "$ADDRESSES" ]; then

From 607568f5da9e6b962c43d59280d72bd678c08afa Mon Sep 17 00:00:00 2001
From: Filippo Tessarotto <zoeslam@gmail.com>
Date: Tue, 7 Feb 2017 12:58:04 +0100
Subject: [PATCH 285/623] Postfix RBL: 554 & SMTP

---
 config/filter.d/postfix-rbl.conf      | 2 +-
 fail2ban/tests/files/logs/postfix-rbl | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf
index c3f8c332..0a9078f0 100644
--- a/config/filter.d/postfix-rbl.conf
+++ b/config/filter.d/postfix-rbl.conf
@@ -12,7 +12,7 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/smtpd
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
 
 ignoreregex =
 
diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl
index eff01bf9..6aeac03b 100644
--- a/fail2ban/tests/files/logs/postfix-rbl
+++ b/fail2ban/tests/files/logs/postfix-rbl
@@ -3,3 +3,6 @@ Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.examp
 
 # failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
 Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
+
+# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" }
+Feb  7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from=<spammer@example.com> to=<goodguy@example.com> proto=SMTP helo=<WIN-5N8GBBS0R5I>

From 1bcf0de7c168da9211982e64b28a839a6078dcf6 Mon Sep 17 00:00:00 2001
From: Jan Grewe <jan@faked.org>
Date: Tue, 7 Feb 2017 21:39:46 +0100
Subject: [PATCH 286/623] Update complain.conf

---
 config/action.d/complain.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index 9934bd94..16117db8 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -62,7 +62,7 @@ actioncheck =
 # Values:  CMD
 #
 actionban = oifs=${IFS}; 
-            REV_IP=$(printf %%s ${IP}. | tac -s.)
+            REV_IP=$(printf %%s <ip>. | tac -s.)
             ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org)
             IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}

From 58c68b75f0b2d879f3fca03cd6e14e66cf5f5804 Mon Sep 17 00:00:00 2001
From: Jan Grewe <jan@faked.org>
Date: Wed, 8 Feb 2017 14:16:13 +0100
Subject: [PATCH 287/623] Remove double-quotes from email addresses

---
 config/action.d/complain.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index 16117db8..3f97d188 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -63,7 +63,7 @@ actioncheck =
 #
 actionban = oifs=${IFS}; 
             REV_IP=$(printf %%s <ip>. | tac -s.)
-            ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org)
+            ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org | tr -d '"')
             IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
             IP=<ip>

From 68d829c1dde3979073414623c9dd4890dd5ee72c Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Tue, 14 Feb 2017 18:12:06 +0800
Subject: [PATCH 288/623] Add a path configuration for Arch Linux

---
 MANIFEST               |  1 +
 config/paths-arch.conf | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 config/paths-arch.conf

diff --git a/MANIFEST b/MANIFEST
index f03f185b..6832f8cf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -148,6 +148,7 @@ config/filter.d/webmin-auth.conf
 config/filter.d/wuftpd.conf
 config/filter.d/xinetd-fail.conf
 config/jail.conf
+config/paths-arch.conf
 config/paths-common.conf
 config/paths-debian.conf
 config/paths-fedora.conf
diff --git a/config/paths-arch.conf b/config/paths-arch.conf
new file mode 100644
index 00000000..270ce65d
--- /dev/null
+++ b/config/paths-arch.conf
@@ -0,0 +1,32 @@
+# Arch
+
+[INCLUDES]
+
+before = paths-common.conf
+
+after  = paths-overrides.local
+
+
+[DEFAULT]
+
+apache_error_log = /var/log/httpd/*error_log
+
+apache_access_log = /var/log/httpd/*access_log
+
+exim_main_log = /var/log/exim/main.log
+
+mysql_log = /var/log/mariadb/mariadb.log
+            /var/log/mysqld.log
+
+roundcube_errors_log = /var/log/roundcubemail/errors
+
+# These services will log to the journal via syslog, so use the journal by
+# default.
+syslog_backend = systemd
+sshd_backend = systemd
+dropbear_backend = systemd
+proftpd_backend = systemd
+pureftpd_backend = systemd
+wuftpd_backend = systemd
+postfix_backend = systemd
+dovecot_backend = systemd

From 861ce4177c07f9771725b2dd94ae8a9b1ed237d7 Mon Sep 17 00:00:00 2001
From: Christoph Theis <theis@gmx.at>
Date: Tue, 14 Feb 2017 18:31:42 +0100
Subject: [PATCH 289/623] #1689: Make lowest rule number in
 action.d/bsd-ipfw.conf configurable

---
 ChangeLog                     |  3 +++
 config/action.d/bsd-ipfw.conf | 10 +++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index b37d7a08..9aa27c09 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -22,6 +22,9 @@ releases.
       (0.10th resp. IPv6 relevant only, amend for gh-1479)
 * config/pathes-freebsd.conf
     - Fixed filenames for apache and nginx log files (gh-1667)
+* action.d/bsd-ipfw.conf
+    - Make the rule number, the action starts looking for a free slot to insert
+      the new rule, configurable (gh-1689)
 
 ### New Features
 * New Actions:
diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf
index 8b0a51aa..65d4294a 100644
--- a/config/action.d/bsd-ipfw.conf
+++ b/config/action.d/bsd-ipfw.conf
@@ -14,7 +14,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
 
 
 # Option:  actionstop
@@ -81,3 +81,11 @@ block = ip
 # Values:  STRING
 #
 blocktype = unreach port
+
+# Option:  lowest_rule_num
+# Notes:   When fail2ban starts with action and there is no rule for the given table yet
+#          then fail2ban will start looking for an empty slot starting with this rule number.
+# Values:  NUM
+lowest_rule_num = 1
+
+

From 40837754c9c12b76c2138cda48e40b06e94cf0dc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Feb 2017 08:45:15 +0100
Subject: [PATCH 290/623] python3.6 compatibility fix

---
 fail2ban/server/actions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 5bd00c85..28c23abd 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -150,7 +150,7 @@ class Actions(JailThread, Mapping):
 				# reload actions after all parameters set via stream:
 				for name, initOpts in self._reload_actions.iteritems():
 					if name in self._actions:
-						self._actions[name].reload(**initOpts if initOpts else {})
+						self._actions[name].reload(**(initOpts if initOpts else {}))
 				# remove obsolete actions (untouched by reload process):
 				delacts = OrderedDict((name, action) for name, action in self._actions.iteritems()
 					if name not in self._reload_actions)

From f35aa6d2589f64acfdea90f9c06d8ec70b3505b4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Feb 2017 08:46:27 +0100
Subject: [PATCH 291/623] coverage: added python3.6

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 3f07cc51..66ed3be2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,7 @@ python:
   - 3.3
   - 3.4
   - 3.5
+  - 3.6
   - pypy3
 before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi

From 63f791688608bf116b3f4a1cca1f479de8b98ef9 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 6 Mar 2016 15:46:52 +0100
Subject: [PATCH 292/623] fix test cases by testing with multi-threaded
 execution (wait for threaded execution done)

(cherry picked from commit 1ec6782f321bf0fca43a2a598cc33f70b39e7f35)

# Conflicts:
#	fail2ban/tests/observertestcase.py (not yet available in 0.10)
---
 fail2ban/tests/action_d/test_smtp.py | 29 ++++++++++++++++++----------
 fail2ban/tests/servertestcase.py     |  3 ++-
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 1dba454c..0aeacbae 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -30,11 +30,14 @@ else:
 
 from ..dummyjail import DummyJail
 
-from ..utils import CONFIG_DIR, asyncserver
-
+from ..utils import CONFIG_DIR, asyncserver, Utils
 
 class TestSMTPServer(smtpd.SMTPServer):
 
+	def __init__(self, *args):
+		smtpd.SMTPServer.__init__(self, *args)
+		self.ready = False
+
 	def process_message(self, peer, mailfrom, rcpttos, data):
 		self.peer = peer
 		self.mailfrom = mailfrom
@@ -42,6 +45,7 @@ class TestSMTPServer(smtpd.SMTPServer):
 		self.org_data = data
 		# replace new line (with tab or space) for possible mime translations (word wrap):
 		self.data = re.sub(r"\n[\t ]", " ", data)
+		self.ready = True
 
 
 class SMTPActionTest(unittest.TestCase):
@@ -78,8 +82,13 @@ class SMTPActionTest(unittest.TestCase):
 		self._active = False
 		self._loop_thread.join()
 
+	def _exec_and_wait(self, doaction):
+		self.smtpd.ready = False
+		doaction()
+		Utils.wait_for(lambda: self.smtpd.ready, 3)
+
 	def testStart(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -87,7 +96,7 @@ class SMTPActionTest(unittest.TestCase):
 			in self.smtpd.data)
 
 	def testStop(self):
-		self.action.stop()
+		self._exec_and_wait(self.action.stop)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -105,7 +114,7 @@ class SMTPActionTest(unittest.TestCase):
 		if restored:
 			aInfo['restored'] = 1
 
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		if restored: # no mail, should raises attribute error:
 			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
 			return
@@ -118,15 +127,15 @@ class SMTPActionTest(unittest.TestCase):
 			"%i attempts" % aInfo['failures'], self.smtpd.data)
 
 		self.action.matches = "matches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['matches'], self.smtpd.data)
 
 		self.action.matches = "ipjailmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipjailmatches'], self.smtpd.data)
 
 		self.action.matches = "ipmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
 	
 	def testBan(self):
@@ -136,14 +145,14 @@ class SMTPActionTest(unittest.TestCase):
 		self._testBan(restored=True)
 
 	def testOptions(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 
 		self.action.fromname = "Test"
 		self.action.fromaddr = "test@example.com"
 		self.action.toaddr = "test@example.com, test2@example.com"
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "test@example.com")
 		self.assertTrue("From: %s <%s>" %
 			(self.action.fromname, self.action.fromaddr) in self.smtpd.data)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index aada699c..9423ee12 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -992,9 +992,10 @@ class LoggingTests(LogCaptureTestCase):
 			badThread = _BadThread()
 			badThread.start()
 			badThread.join()
-			self.assertLogged("Unhandled exception")
+			self.assertTrue( Utils.wait_for( lambda: len(x) and self._is_logged("Unhandled exception"), 3) )
 		finally:
 			sys.__excepthook__ = prev_exchook
+		self.assertLogged("Unhandled exception")
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
 

From 01db0b502854f23ae9bd23906ccc0e48b80f0465 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sat, 21 Jan 2017 17:54:01 +0100
Subject: [PATCH 293/623] small performance fix after merge with 0.10

(cherry picked from commit 8e2711681cf0f65ca5d3ef6831246ec5d4973c21)
---
 fail2ban/tests/action_d/test_smtp.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 0aeacbae..8334f948 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -82,10 +82,11 @@ class SMTPActionTest(unittest.TestCase):
 		self._active = False
 		self._loop_thread.join()
 
-	def _exec_and_wait(self, doaction):
+	def _exec_and_wait(self, doaction, timeout=3, short=False):
+		if short: timeout /= 25
 		self.smtpd.ready = False
 		doaction()
-		Utils.wait_for(lambda: self.smtpd.ready, 3)
+		Utils.wait_for(lambda: self.smtpd.ready, timeout)
 
 	def testStart(self):
 		self._exec_and_wait(self.action.start)
@@ -114,7 +115,7 @@ class SMTPActionTest(unittest.TestCase):
 		if restored:
 			aInfo['restored'] = 1
 
-		self._exec_and_wait(lambda: self.action.ban(aInfo))
+		self._exec_and_wait(lambda: self.action.ban(aInfo), short=restored)
 		if restored: # no mail, should raises attribute error:
 			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
 			return

From cf53a834f7b1ed623e5ff81cb6a4ffedcc1d48e4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Feb 2017 18:46:00 +0100
Subject: [PATCH 294/623] python-3.6 compatibility: - dynamical string
 replacement within call of regexp.sub fixed with lambda-replacement
 (otherwise "sre_constants.error: bad escape \s at position"); - optional
 arguments (3.6 has more agrs by calling of SMTPServer.process_message); -
 implicit convert byte to string, because python3.6 fails on binary data
 (test_smtp).

---
 fail2ban/server/datedetector.py      | 2 +-
 fail2ban/tests/action_d/test_smtp.py | 8 ++++----
 fail2ban/tests/servertestcase.py     | 2 +-
 fail2ban/tests/utils.py              | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index cccbf71a..dd5d198f 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -284,7 +284,7 @@ class DateDetector(object):
 			if preMatch is not None:
 				# get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}:
 				template = _getAnchoredTemplate(template,
-					wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch))
+					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: s, preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
 
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index 8334f948..bf8e83a2 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -30,7 +30,7 @@ else:
 
 from ..dummyjail import DummyJail
 
-from ..utils import CONFIG_DIR, asyncserver, Utils
+from ..utils import CONFIG_DIR, asyncserver, Utils, uni_decode
 
 class TestSMTPServer(smtpd.SMTPServer):
 
@@ -38,13 +38,13 @@ class TestSMTPServer(smtpd.SMTPServer):
 		smtpd.SMTPServer.__init__(self, *args)
 		self.ready = False
 
-	def process_message(self, peer, mailfrom, rcpttos, data):
+	def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
 		self.peer = peer
 		self.mailfrom = mailfrom
 		self.rcpttos = rcpttos
 		self.org_data = data
-		# replace new line (with tab or space) for possible mime translations (word wrap):
-		self.data = re.sub(r"\n[\t ]", " ", data)
+		# replace new line (with tab or space) for possible mime translations (word wrap),
+		self.data = re.sub(r"\n[\t ]", " ", uni_decode(data))
 		self.ready = True
 
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 9423ee12..2f1f5451 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1647,7 +1647,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				r' echo mail \1 ) | cat', realCmd)
 			# replace abuse retrieving (possible no-network):
 			realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+',
-				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
+				lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
 			# execute action:
 			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index ae543e3d..78a42d09 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -37,7 +37,7 @@ import unittest
 from cStringIO import StringIO
 from functools import wraps
 
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode
 from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils

From 2d12349eaccb221e6ffc8aea80a366b5ebaea579 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 15 Feb 2017 20:30:48 +0100
Subject: [PATCH 295/623] Update servertestcase.py

Make the test-case gh-1685 compliant
---
 fail2ban/tests/servertestcase.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index aada699c..c5b997f3 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1644,9 +1644,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			# replace pipe to mail with pipe to cat:
 			realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)',
 				r' echo mail \1 ) | cat', realCmd)
-			# replace abuse retrieving (possible no-network):
-			realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+',
-				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
+			# replace abuse retrieving (possible no-network), just replace first occurrence of 'dig...':
+			realCmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+',
+				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd, 1)
 			# execute action:
 			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
 

From fc315be4ea88c3619f984542b21c95820f53d87b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 13:32:56 +0100
Subject: [PATCH 296/623] try to parse and interpolate all options in section
 "Definition" (section "Init" no more needed)

---
 fail2ban/client/configreader.py | 22 ++++++++++++++++++----
 fail2ban/server/action.py       |  6 +++++-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 7840cd12..b44a8c57 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -221,10 +221,14 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
+	def getOptions(self, sec, options, pOptions=None, 
+		allOpts=None, shouldExist=False
+	):
 		values = dict()
 		if pOptions is None:
 			pOptions = {}
+
+		# Get only specified options:
 		for optname in options:
 			if isinstance(options, (list,tuple)):
 				if len(optname) > 2:
@@ -261,6 +265,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
 							"'. Using default one: '" + repr(optvalue) + "'")
 				values[optname] = optvalue
+
+		# Fill all option of the section (used for replacement):
+		if allOpts is not None and self.has_section(sec):
+			for optname in self.options(sec):
+				v = values.get(optname)
+				if v is None:
+					v = self.get(sec, optname, vars=pOptions)
+				allOpts[optname] = v
+
 		return values
 
 
@@ -310,9 +323,9 @@ class DefinitionInitConfigReader(ConfigReader):
 			if not pOpts:
 				pOpts = dict()
 			pOpts = _merge_dicts(pOpts, self._initOpts)
+		self._allOpts = dict()
 		self._opts = ConfigReader.getOptions(
-			self, "Definition", self._configOpts, pOpts)
-		
+			self, "Definition", self._configOpts, pOpts, allOpts=self._allOpts)
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
 				v = self.get("Init", opt)
@@ -338,7 +351,8 @@ class DefinitionInitConfigReader(ConfigReader):
 				n, cond = cond.groups()
 				ignore.add(n)
 		# substiture options already specified direct:
-		opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore)
+		opts = CommandAction.substituteRecursiveTags(combinedopts,
+			ignore=ignore, addtags=self._allOpts)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
 		return opts
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 46a19cd1..69cd84a3 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -365,7 +365,9 @@ class CommandAction(ActionBase):
 		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
-	def substituteRecursiveTags(cls, inptags, conditional='', ignore=()):
+	def substituteRecursiveTags(cls, inptags, conditional='', 
+		ignore=(), addtags={}
+	):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -420,6 +422,8 @@ class CommandAction(ActionBase):
 						repl = tags.get(found_tag + '?' + conditional)
 						if repl is None:
 							repl = tags.get(found_tag)
+							if repl is None:
+								repl = addtags.get(found_tag)
 					if repl is None:
 						# Escaped or missing tags - just continue on searching after end of match
 						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell

From 4bf09bf2972df852133f64a3645eb391c16f51c9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 13:38:20 +0100
Subject: [PATCH 297/623] provides new tag `<ip-rev>` for PTR reversed
 representation of IP address; [action.d/complain.conf] fixed using this new
 tag;

---
 config/action.d/complain.conf    | 17 +++++++++++++----
 fail2ban/server/actions.py       | 16 +++++++++-------
 fail2ban/server/ipdns.py         |  6 +++---
 fail2ban/tests/servertestcase.py | 25 ++++++++++++++++++++-----
 4 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index 3f97d188..1f74d635 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -34,6 +34,9 @@ before = helpers-common.conf
 
 [Definition]
 
+# Used in test cases for coverage internal transformations
+debug = 0
+
 # bypass ban/unban for restored tickets
 norestored = 1
 
@@ -61,9 +64,10 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; 
-            REV_IP=$(printf %%s <ip>. | tac -s.)
-            ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org | tr -d '"')
+actionban = oifs=${IFS};
+            RESOLVER_ADDR="%(addr_resolver)s"
+            if [ "<debug>" -gt 0 ]; then echo "try to resolve $RESOLVER_ADDR"; fi
+            ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | tr -d '"')
             IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
             IP=<ip>
@@ -82,7 +86,12 @@ actionban = oifs=${IFS};
 #
 actionunban =
 
-[Init]
+# Server as resolver used in dig command
+#
+addr_resolver = <ip-rev>abuse-contacts.abusix.org
+
+# Default message used for abuse content
+#
 message = Dear Sir/Madam,\n\nWe have detected abuse from the IP address $IP, which according to a abusix.com is on your network. We would appreciate if you would investigate and take action as appropriate.\n\nLog lines are given below, but please ask if you require any further information.\n\n(If you are not the correct person to contact about this please accept our apologies - your e-mail address was extracted from the whois record by an automated process.)\n\n This mail was generated by Fail2Ban.\nThe recipient address of this report was provided by the Abuse Contact DB by abusix.com. abusix.com does not maintain the content of the database. All information which we pass out, derives from the RIR databases and is processed for ease of use. If you want to change or report non working abuse contacts please contact the appropriate RIR. If you have any further question, contact abusix.com directly via email (info@abusix.com). Information about the Abuse Contact Database can be found here: https://abusix.com/global-reporting/abuse-contact-db\nabusix.com is neither responsible nor liable for the content or accuracy of this message.\n
 
 # Path to the log files which contain relevant lines for the abuser IP
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 28c23abd..e2046e15 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -341,15 +341,17 @@ class Actions(JailThread, Mapping):
 			ticket = self._jail.getFailTicket()
 			if not ticket:
 				break
-			aInfo = CallingMap()
 			bTicket = BanManager.createBanTicket(ticket)
 			ip = bTicket.getIP()
-			aInfo["ip"] = ip
-			aInfo["failures"] = bTicket.getAttempt()
-			aInfo["time"] = bTicket.getTime()
-			aInfo["matches"] = "\n".join(bTicket.getMatches())
-			# to bypass actions, that should not be executed for restored tickets
-			aInfo["restored"] = 1 if ticket.restored else 0
+			aInfo = CallingMap({
+				"ip"			:	ip,
+				"ip-rev"	:	lambda: ip.getPTR(''),
+				"failures":	bTicket.getAttempt(),
+				"time"		:	bTicket.getTime(),
+				"matches"	:	"\n".join(bTicket.getMatches()),
+				# to bypass actions, that should not be executed for restored tickets
+				"restored":	(1 if ticket.restored else 0)
+			})
 			if self._jail.database is not None:
 				mi4ip = lambda overalljails=False, self=self, \
 					mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index f8db6a04..a5625629 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -346,7 +346,7 @@ class IPAddr(object):
 		
 		return socket.inet_ntop(self._family, binary) + add
 
-	def getPTR(self, suffix=""):
+	def getPTR(self, suffix=None):
 		""" return the DNS PTR string of the provided IP address object
 
 			If "suffix" is provided it will be appended as the second and top
@@ -356,11 +356,11 @@ class IPAddr(object):
 		"""
 		if self.isIPv4:
 			exploded_ip = self.ntoa.split(".")
-			if not suffix:
+			if suffix is None:
 				suffix = "in-addr.arpa."
 		elif self.isIPv6:
 			exploded_ip = self.hexdump
-			if not suffix:
+			if suffix is None:
 				suffix = "ip6.arpa."
 		else:
 			return ""
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index f5541ed0..fce03573 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1678,18 +1678,29 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				('j-complain-abuse', 
 					'complain['
 					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' +
+					  # test reverse ip:
+					  'debug=1,' +
 						# 2 logs to test grep from multiple logs:
 					  'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' +
 				    '         ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", '
 					  ']',
 				{
 					'ip4-ban': (
+						# test reverse ip:
+						'try to resolve 10.124.142.87.abuse-contacts.abusix.org',
 						'Lines containing failures of 87.142.124.10 (max 2)',
 						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
 						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
 						# both abuse mails should be separated with space:
 						'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
 					),
+					'ip6-ban': (
+						# test reverse ip:
+						'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org',
+						'Lines containing failures of 2001:db8::1 (max 2)',
+						# both abuse mails should be separated with space:
+						'mail -s Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+					),
 				}),
 			)
 			server = TestServer()
@@ -1710,6 +1721,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 			jails = server._Server__jails
 
+			ipv4 = IPAddr('87.142.124.10')
+			ipv6 = IPAddr('2001:db8::1');
 			for jail, act, tests in testJailsActions:
 				# print(jail, jails[jail])
 				for a in jails[jail].actions:
@@ -1720,8 +1733,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# wrap default command processor:
 					action.executeCmd = self._executeMailCmd
 					# test ban :
-					self.pruneLog('# === ban ===')
-					action.ban({'ip': IPAddr('87.142.124.10'), 
-						'failures': 100,
-					})
-					self.assertLogged(*tests['ip4-ban'], all=True)
+					for (test, ip) in (('ip4-ban', ipv4), ('ip6-ban', ipv6)):
+						if not tests.get(test): continue
+						self.pruneLog('# === %s ===' % test)
+						ticket = _actions.CallingMap({
+							'ip': ip, 'ip-rev': lambda: ip.getPTR(''), 'failures': 100,})
+						action.ban(ticket)
+						self.assertLogged(*tests[test], all=True)

From 3fae8a7e432217c5ff650ef156a35663e4ccc78d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 14:53:57 +0100
Subject: [PATCH 298/623] amend to fc315be4ea88c3619f984542b21c95820f53d87b:
 parse and interpolate all options in section "Definition" (section "Init" no
 more needed), because of better performance with this solution;

---
 fail2ban/client/configreader.py | 33 +++++++++++++++++----------------
 fail2ban/server/action.py       |  9 +++++----
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index b44a8c57..8098433e 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -221,13 +221,10 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	# Or it is a dict:
 	#  {name: [type, default], ...}
 	
-	def getOptions(self, sec, options, pOptions=None, 
-		allOpts=None, shouldExist=False
-	):
+	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 		values = dict()
 		if pOptions is None:
 			pOptions = {}
-
 		# Get only specified options:
 		for optname in options:
 			if isinstance(options, (list,tuple)):
@@ -265,15 +262,6 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
 							"'. Using default one: '" + repr(optvalue) + "'")
 				values[optname] = optvalue
-
-		# Fill all option of the section (used for replacement):
-		if allOpts is not None and self.has_section(sec):
-			for optname in self.options(sec):
-				v = values.get(optname)
-				if v is None:
-					v = self.get(sec, optname, vars=pOptions)
-				allOpts[optname] = v
-
 		return values
 
 
@@ -293,6 +281,8 @@ class DefinitionInitConfigReader(ConfigReader):
 		self.setFile(file_)
 		self.setJailName(jailName)
 		self._initOpts = initOpts
+		self._pOpts = dict()
+		self._defCache = dict()
 	
 	def setFile(self, fileName):
 		self._file = fileName
@@ -323,9 +313,9 @@ class DefinitionInitConfigReader(ConfigReader):
 			if not pOpts:
 				pOpts = dict()
 			pOpts = _merge_dicts(pOpts, self._initOpts)
-		self._allOpts = dict()
 		self._opts = ConfigReader.getOptions(
-			self, "Definition", self._configOpts, pOpts, allOpts=self._allOpts)
+			self, "Definition", self._configOpts, pOpts)
+		self._pOpts = pOpts
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
 				v = self.get("Init", opt)
@@ -337,6 +327,17 @@ class DefinitionInitConfigReader(ConfigReader):
 	def _convert_to_boolean(self, value):
 		return value.lower() in ("1", "yes", "true", "on")
 	
+	def getCombOption(self, optname):
+		try:
+			return self._defCache[optname]
+		except KeyError:
+			try:
+				v = self.get("Definition", optname, vars=self._pOpts)
+			except (NoSectionError, NoOptionError, ValueError):
+				v = None
+			self._defCache[optname] = v
+			return v
+
 	def getCombined(self, ignore=()):
 		combinedopts = self._opts
 		ignore = set(ignore).copy()
@@ -352,7 +353,7 @@ class DefinitionInitConfigReader(ConfigReader):
 				ignore.add(n)
 		# substiture options already specified direct:
 		opts = CommandAction.substituteRecursiveTags(combinedopts,
-			ignore=ignore, addtags=self._allOpts)
+			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
 		return opts
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 69cd84a3..d14c9aab 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -366,7 +366,7 @@ class CommandAction(ActionBase):
 
 	@classmethod
 	def substituteRecursiveTags(cls, inptags, conditional='', 
-		ignore=(), addtags={}
+		ignore=(), addrepl=None
 	):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
@@ -419,11 +419,12 @@ class CommandAction(ActionBase):
 							(tag, found_tag, refCounts, value))
 					repl = None
 					if found_tag not in cls._escapedTags:
-						repl = tags.get(found_tag + '?' + conditional)
+						if conditional:
+							repl = tags.get(found_tag + '?' + conditional)
 						if repl is None:
 							repl = tags.get(found_tag)
-							if repl is None:
-								repl = addtags.get(found_tag)
+							if repl is None and addrepl is not None:
+								repl = addrepl(found_tag)
 					if repl is None:
 						# Escaped or missing tags - just continue on searching after end of match
 						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell

From a6318b159b0c2db5405aa525882e208aac240ead Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 16:00:24 +0100
Subject: [PATCH 299/623] substituteRecursiveTags optimization + moved in
 helpers facilities (because currently used commonly in server and in client)

---
 fail2ban/client/actionreader.py  |  4 +-
 fail2ban/client/configreader.py  | 10 +++-
 fail2ban/helpers.py              | 99 ++++++++++++++++++++++++++++++++
 fail2ban/server/action.py        | 98 +------------------------------
 fail2ban/tests/actiontestcase.py | 40 ++++++-------
 5 files changed, 131 insertions(+), 120 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index 55ceda93..0fd55f41 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -28,6 +28,7 @@ import os
 
 from .configreader import DefinitionInitConfigReader
 from ..helpers import getLogger
+from ..server.action import CommandAction
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -69,7 +70,8 @@ class ActionReader(DefinitionInitConfigReader):
 		return self._name
 
 	def convert(self):
-		opts = self.getCombined(ignore=('timeout', 'bantime'))
+		opts = self.getCombined(
+			ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
 		# type-convert only after combined (otherwise boolean converting prevents substitution):
 		if opts.get('norestored'):
 			opts['norestored'] = self._convert_to_boolean(opts['norestored'])
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 8098433e..2caa97ce 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -29,8 +29,7 @@ import os
 from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
-from ..helpers import getLogger
-from ..server.action import CommandAction
+from ..helpers import getLogger, substituteRecursiveTags
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -328,6 +327,11 @@ class DefinitionInitConfigReader(ConfigReader):
 		return value.lower() in ("1", "yes", "true", "on")
 	
 	def getCombOption(self, optname):
+		"""Get combined definition option (as string) using pre-set and init
+		options as preselection (values with higher precedence as specified in section).
+
+		Can be used only after calling of getOptions.
+		"""
 		try:
 			return self._defCache[optname]
 		except KeyError:
@@ -352,7 +356,7 @@ class DefinitionInitConfigReader(ConfigReader):
 				n, cond = cond.groups()
 				ignore.add(n)
 		# substiture options already specified direct:
-		opts = CommandAction.substituteRecursiveTags(combinedopts,
+		opts = substituteRecursiveTags(combinedopts, 
 			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:
 			raise ValueError('recursive tag definitions unable to be resolved')
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 4f4426b0..1e5a053b 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -200,6 +200,105 @@ else:
 				raise
 			return uni_decode(x, enc, 'replace')
 
+
+#
+# Following facilities used for safe recursive interpolation of
+# tags (<tag>) in tagged options.
+#
+
+# max tag replacement count:
+MAX_TAG_REPLACE_COUNT = 10
+
+# compiled RE for tag name (replacement name) 
+TAG_CRE = re.compile(r'<([^ <>]+)>')
+
+def substituteRecursiveTags(inptags, conditional='', 
+	ignore=(), addrepl=None
+):
+	"""Sort out tag definitions within other tags.
+	Since v.0.9.2 supports embedded interpolation (see test cases for examples).
+
+	so:		becomes:
+	a = 3		a = 3
+	b = <a>_3	b = 3_3
+
+	Parameters
+	----------
+	inptags : dict
+		Dictionary of tags(keys) and their values.
+
+	Returns
+	-------
+	dict
+		Dictionary of tags(keys) and their values, with tags
+		within the values recursively replaced.
+	"""
+	# copy return tags dict to prevent modifying of inptags:
+	tags = inptags.copy()
+	t = TAG_CRE
+	ignore = set(ignore)
+	done = set()
+	# repeat substitution while embedded-recursive (repFlag is True)
+	while True:
+		repFlag = False
+		# substitute each value:
+		for tag in tags.iterkeys():
+			# ignore escaped or already done (or in ignore list):
+			if tag in ignore or tag in done: continue
+			value = orgval = str(tags[tag])
+			# search and replace all tags within value, that can be interpolated using other tags:
+			m = t.search(value)
+			refCounts = {}
+			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
+			while m:
+				found_tag = m.group(1)
+				# don't replace tags that should be currently ignored (pre-replacement):
+				if found_tag in ignore: 
+					m = t.search(value, m.end())
+					continue
+				#logSys.log(5, 'found: %s' % found_tag)
+				if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
+					# recursive definitions are bad
+					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
+					raise ValueError(
+						"properties contain self referencing definitions "
+						"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
+						(tag, found_tag, refCounts, value))
+				repl = None
+				if conditional:
+					repl = tags.get(found_tag + '?' + conditional)
+				if repl is None:
+					repl = tags.get(found_tag)
+					# try to find tag using additional replacement (callable):
+					if repl is None and addrepl is not None:
+						repl = addrepl(found_tag)
+				if repl is None:
+					# Missing tags - just continue on searching after end of match
+					# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
+					# constructs like <STDIN>.
+					m = t.search(value, m.end())
+					continue
+				value = value.replace('<%s>' % found_tag, repl)
+				#logSys.log(5, 'value now: %s' % value)
+				# increment reference count:
+				refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+				# the next match for replace:
+				m = t.search(value, m.start())
+			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
+			# was substituted?
+			if orgval != value:
+				# check still contains any tag - should be repeated (possible embedded-recursive substitution):
+				if t.search(value):
+					repFlag = True
+				tags[tag] = value
+			# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
+			if '<' not in value: done.add(tag)
+		# stop interpolation, if no replacements anymore:
+		if not repFlag:
+			break
+	return tags
+
+
 class BgService(object):
 	"""Background servicing
 
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index d14c9aab..2f7f423c 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -23,7 +23,6 @@ __license__ = "GPL"
 
 import logging
 import os
-import re
 import signal
 import subprocess
 import tempfile
@@ -35,7 +34,7 @@ from collections import MutableMapping
 from .ipdns import asip
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger
+from ..helpers import getLogger, substituteRecursiveTags
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -46,12 +45,6 @@ _cmd_lock = threading.Lock()
 # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
 allowed_ipv6 = True
 
-# max tag replacement count:
-MAX_TAG_REPLACE_COUNT = 10
-
-# compiled RE for tag name (replacement name) 
-TAG_CRE = re.compile(r'<([^ <>]+)>')
-
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -364,93 +357,6 @@ class CommandAction(ActionBase):
 		"""
 		return self._executeOperation('<actionreload>', 'reloading')
 
-	@classmethod
-	def substituteRecursiveTags(cls, inptags, conditional='', 
-		ignore=(), addrepl=None
-	):
-		"""Sort out tag definitions within other tags.
-		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
-
-		so:		becomes:
-		a = 3		a = 3
-		b = <a>_3	b = 3_3
-
-		Parameters
-		----------
-		inptags : dict
-			Dictionary of tags(keys) and their values.
-
-		Returns
-		-------
-		dict
-			Dictionary of tags(keys) and their values, with tags
-			within the values recursively replaced.
-		"""
-		# copy return tags dict to prevent modifying of inptags:
-		tags = inptags.copy()
-		t = TAG_CRE
-		ignore = set(ignore)
-		done = cls._escapedTags.copy() | ignore
-		# repeat substitution while embedded-recursive (repFlag is True)
-		while True:
-			repFlag = False
-			# substitute each value:
-			for tag in tags.iterkeys():
-				# ignore escaped or already done (or in ignore list):
-				if tag in done: continue
-				value = orgval = str(tags[tag])
-				# search and replace all tags within value, that can be interpolated using other tags:
-				m = t.search(value)
-				refCounts = {}
-				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
-				while m:
-					found_tag = m.group(1)
-					# don't replace tags that should be currently ignored (pre-replacement):
-					if found_tag in ignore: 
-						m = t.search(value, m.end())
-						continue
-					#logSys.log(5, 'found: %s' % found_tag)
-					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
-						# recursive definitions are bad
-						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
-						raise ValueError(
-							"properties contain self referencing definitions "
-							"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-							(tag, found_tag, refCounts, value))
-					repl = None
-					if found_tag not in cls._escapedTags:
-						if conditional:
-							repl = tags.get(found_tag + '?' + conditional)
-						if repl is None:
-							repl = tags.get(found_tag)
-							if repl is None and addrepl is not None:
-								repl = addrepl(found_tag)
-					if repl is None:
-						# Escaped or missing tags - just continue on searching after end of match
-						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
-						# constructs like <STDIN>.
-						m = t.search(value, m.end())
-						continue
-					value = value.replace('<%s>' % found_tag, repl)
-					#logSys.log(5, 'value now: %s' % value)
-					# increment reference count:
-					refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
-					# the next match for replace:
-					m = t.search(value, m.start())
-				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
-				# was substituted?
-				if orgval != value:
-					# check still contains any tag - should be repeated (possible embedded-recursive substitution):
-					if t.search(value):
-						repFlag = True
-					tags[tag] = value
-				# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
-				if '<' not in value: done.add(tag)
-			# stop interpolation, if no replacements anymore:
-			if not repFlag:
-				break
-		return tags
-
 	@staticmethod
 	def escapeTag(value):
 		"""Escape characters which may be used for command injection.
@@ -501,7 +407,7 @@ class CommandAction(ActionBase):
 				return string
 		# replace:
 		string = query
-		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
+		aInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
 				value = aInfo.get(tag + '?' + conditional)
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 403f7ea6..8a74201c 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -29,7 +29,7 @@ import tempfile
 import time
 import unittest
 
-from ..server.action import CommandAction, CallingMap
+from ..server.action import CommandAction, CallingMap, substituteRecursiveTags
 from ..server.actions import OrderedDict
 from ..server.utils import Utils
 
@@ -56,30 +56,30 @@ class CommandActionTest(LogCaptureTestCase):
 		}
 		# Recursion is bad
 		self.assertRaises(ValueError,
-			lambda: CommandAction.substituteRecursiveTags({'A': '<A>'}))
+			lambda: substituteRecursiveTags({'A': '<A>'}))
 		self.assertRaises(ValueError,
-			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
+			lambda: substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
 		self.assertRaises(ValueError,
-			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
+			lambda: substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# Unresolveable substition
 		self.assertRaises(ValueError,
-			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
+			lambda: substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
 		self.assertRaises(ValueError,
-			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+			lambda: substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
 		# We need here an ordered, because the sequence of iteration is very important for this test
 		if OrderedDict:
 			# No cyclic recursion, just multiple replacement of tag <T>, should be successful:
-			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+			self.assertEqual(substituteRecursiveTags( OrderedDict(
 					(('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>')))
 				), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'}
 			)
 			# No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful:
-			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(
+			self.assertEqual(substituteRecursiveTags( OrderedDict(
 				  (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>')))
 				), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'}
 			)
 			# No cyclic recursion, just multiple replacement of same tags, should be successful:
-			self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((
+			self.assertEqual(substituteRecursiveTags( OrderedDict((
 					('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'),
 					('ipmset', 'f2b-<name>'),
 					('name', 'any'),
@@ -111,42 +111,42 @@ class CommandActionTest(LogCaptureTestCase):
 				))
 			)
 			# Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle:
-			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+			self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict((
 					('A', '<<B><C>>'),
 					('B', 'D'), ('C', 'E'),
 					('DE', 'cycle <A>'),
 			)) ))
-			self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict((
+			self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict((
 					('DE', 'cycle <A>'),
 					('A', '<<B><C>>'),
 					('B', 'D'), ('C', 'E'),
 			)) ))
 			
 		# missing tags are ok
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
+		self.assertEqual(substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
+		self.assertEqual(substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
+		self.assertEqual(substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
 		# Escaped tags should be ignored
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<matches> <B>', 'B': 'cool'}), {'A': '<matches> cool', 'B': 'cool'})
+		self.assertEqual(substituteRecursiveTags({'A': '<matches> <B>', 'B': 'cool'}), {'A': '<matches> cool', 'B': 'cool'})
 		# Multiple stuff on same line is ok
-		self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP> evilperson=<honeypot>', 'honeypot': 'pokie', 'ignoreregex': ''}),
+		self.assertEqual(substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP> evilperson=<honeypot>', 'honeypot': 'pokie', 'ignoreregex': ''}),
 								{ 'failregex': "to=pokie fromip=<IP> evilperson=pokie",
 									'honeypot': 'pokie',
 									'ignoreregex': '',
 								})
 		# rest is just cool
-		self.assertEqual(CommandAction.substituteRecursiveTags(aInfo),
+		self.assertEqual(substituteRecursiveTags(aInfo),
 								{ 'HOST': "192.0.2.0",
 									'ABC': '123 192.0.2.0',
 									'xyz': '890 123 192.0.2.0',
 								})
 		# obscure embedded case
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4'}),
+		self.assertEqual(substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4'}),
 						 {'A': '<IPV4HOST>', 'PREF': 'IPV4'})
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}),
+		self.assertEqual(substituteRecursiveTags({'A': '<<PREF>HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}),
 						 {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'})
 		# more embedded within a string and two interpolations
-		self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A <IP<PREF>HOST> B IP<PREF> C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}),
+		self.assertEqual(substituteRecursiveTags({'A': 'A <IP<PREF>HOST> B IP<PREF> C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}),
 						 {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'})
 
 	def testReplaceTag(self):

From 9ebf70cd6a90143342733e4e8af85be3455b2795 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 18:25:34 +0100
Subject: [PATCH 300/623] Safer, more stable and faster replaceTag
 interpolation (switched from cycle over all tags to re.sub with callable)

---
 fail2ban/server/action.py        | 86 +++++++++++++++++++++++---------
 fail2ban/tests/actiontestcase.py |  6 +--
 2 files changed, 65 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 2f7f423c..5acc98ad 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -34,7 +34,7 @@ from collections import MutableMapping
 from .ipdns import asip
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, substituteRecursiveTags
+from ..helpers import getLogger, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -399,33 +399,71 @@ class CommandAction(ActionBase):
 		str
 			`query` string with tags replaced.
 		"""
+		if '<' not in query: return query
+
 		# use cache if allowed:
 		if cache is not None:
 			ckey = (query, conditional)
-			string = cache.get(ckey)
-			if string is not None:
-				return string
-		# replace:
-		string = query
-		aInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
-		for tag in aInfo:
-			if "<%s>" % tag in query:
-				value = aInfo.get(tag + '?' + conditional)
-				if value is None:
-					value = aInfo.get(tag)
-				value = str(value)			  # assure string
-				if tag in cls._escapedTags:
-					# That one needs to be escaped since its content is
-					# out of our control
-					value = cls.escapeTag(value)
-				string = string.replace('<' + tag + '>', value)
-		# New line, space
-		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
-		# cache if properties:
+			value = cache.get(ckey)
+			if value is not None:
+				return value
+
+		# first try get cached tags dictionary:
+		subInfo = csubkey = None
 		if cache is not None:
-			cache[ckey] = string
+			csubkey = ('subst-tags', id(aInfo), conditional)
+			subInfo = cache.get(csubkey)
+		# interpolation of dictionary:
+		if subInfo is None:
+			subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
+		# New line, space
+		for (tag, value) in (("br", '\n'), ("sp", " ")):
+			if subInfo.get(tag) is None: subInfo[tag] = value
+		# cache if possible:
+		if csubkey is not None:
+			cache[csubkey] = subInfo
+
+		# substitution callable, used by interpolation of each tag
+		repeatSubst = {}
+		def substVal(m):
+			tag = m.group(1)			# tagname from match
+			value = None
+			if conditional:
+				value = subInfo.get(tag + '?' + conditional)
+			if value is None:
+				value = subInfo.get(tag)
+				if value is None:
+					return m.group()	# fallback (no replacement)
+			value = str(value)		# assure string
+			if tag in cls._escapedTags:
+				# That one needs to be escaped since its content is
+				# out of our control
+				value = cls.escapeTag(value)
+			# possible contains tags:
+			if '<' in value:
+				repeatSubst[1] = True
+			return value
+
+		# interpolation of query:
+		count = MAX_TAG_REPLACE_COUNT + 1
+		while True:
+			repeatSubst = {}
+			value = TAG_CRE.sub(substVal, query)
+			# possible recursion ?
+			if not repeatSubst or value == query: break
+			query = value
+			count -= 1
+			if count <= 0: # pragma: no cover - almost impossible (because resolved above)
+				raise ValueError(
+					"unexpected too long replacement interpolation, "
+					"possible self referencing definitions in query: %s" % (query,))
+
+
+		# cache if possible:
+		if cache is not None:
+			cache[ckey] = value
 		#
-		return string
+		return value
 
 	def _processCmd(self, cmd, aInfo=None, conditional=''):
 		"""Executes a command with preliminary checks and substitutions.
@@ -491,7 +529,7 @@ class CommandAction(ActionBase):
 		realCmd = self.replaceTag(cmd, self._properties, 
 			conditional=conditional, cache=self.__substCache)
 
-		# Replace tags
+		# Replace dynamical tags (don't use cache here)
 		if aInfo is not None:
 			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
 		else:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 8a74201c..38f576b1 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -217,10 +217,10 @@ class CommandActionTest(LogCaptureTestCase):
 				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
 				"Text 890-567 text 567 '567'")
-		self.assertEqual(len(cache) if cache is not None else -1, 3)
+		self.assertTrue(len(cache) >= 3)
 		# set one parameter - internal properties and cache should be reseted:
 		setattr(self.__action, 'xyz', "000-<abc>")
-		self.assertEqual(len(cache) if cache is not None else -1, 0)
+		self.assertEqual(len(cache), 0)
 		# test againg, should have 000 instead of 890:
 		for i in range(2):
 			self.assertEqual(
@@ -235,7 +235,7 @@ class CommandActionTest(LogCaptureTestCase):
 				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
 					conditional="family=inet6", cache=cache),
 				"Text 000-567 text 567 '567'")
-		self.assertEqual(len(cache), 3)
+		self.assertTrue(len(cache) >= 3)
 
 
 	def testExecuteActionBan(self):

From a8c0cec4ac1c91b821d3fc80e0aac91a7b12902a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 21:00:48 +0100
Subject: [PATCH 301/623] small amend with several fixes and test coverage

---
 fail2ban/helpers.py              | 35 +++++++++++++++++---------------
 fail2ban/server/action.py        | 30 +++++++++++++++++++--------
 fail2ban/tests/actiontestcase.py | 32 +++++++++++++++++++++++++++--
 3 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 1e5a053b..2407df1f 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -233,9 +233,11 @@ def substituteRecursiveTags(inptags, conditional='',
 		Dictionary of tags(keys) and their values, with tags
 		within the values recursively replaced.
 	"""
+	#logSys = getLogger("fail2ban")
+	tre_search = TAG_CRE.search
 	# copy return tags dict to prevent modifying of inptags:
 	tags = inptags.copy()
-	t = TAG_CRE
+	# init:
 	ignore = set(ignore)
 	done = set()
 	# repeat substitution while embedded-recursive (repFlag is True)
@@ -247,48 +249,49 @@ def substituteRecursiveTags(inptags, conditional='',
 			if tag in ignore or tag in done: continue
 			value = orgval = str(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
-			m = t.search(value)
+			m = tre_search(value)
 			refCounts = {}
 			#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 			while m:
-				found_tag = m.group(1)
+				# found replacement tag:
+				rtag = m.group(1)
 				# don't replace tags that should be currently ignored (pre-replacement):
-				if found_tag in ignore: 
-					m = t.search(value, m.end())
+				if rtag in ignore: 
+					m = tre_search(value, m.end())
 					continue
-				#logSys.log(5, 'found: %s' % found_tag)
-				if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
+				#logSys.log(5, 'found: %s' % rtag)
+				if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT:
 					# recursive definitions are bad
 					#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
 					raise ValueError(
 						"properties contain self referencing definitions "
 						"and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % 
-						(tag, found_tag, refCounts, value))
+						(tag, rtag, refCounts, value))
 				repl = None
 				if conditional:
-					repl = tags.get(found_tag + '?' + conditional)
+					repl = tags.get(rtag + '?' + conditional)
 				if repl is None:
-					repl = tags.get(found_tag)
+					repl = tags.get(rtag)
 					# try to find tag using additional replacement (callable):
 					if repl is None and addrepl is not None:
-						repl = addrepl(found_tag)
+						repl = addrepl(rtag)
 				if repl is None:
 					# Missing tags - just continue on searching after end of match
 					# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
 					# constructs like <STDIN>.
-					m = t.search(value, m.end())
+					m = tre_search(value, m.end())
 					continue
-				value = value.replace('<%s>' % found_tag, repl)
+				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
-				refCounts[found_tag] = refCounts.get(found_tag, 0) + 1
+				refCounts[rtag] = refCounts.get(rtag, 0) + 1
 				# the next match for replace:
-				m = t.search(value, m.start())
+				m = tre_search(value, m.start())
 			#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 			# was substituted?
 			if orgval != value:
 				# check still contains any tag - should be repeated (possible embedded-recursive substitution):
-				if t.search(value):
+				if tre_search(value):
 					repFlag = True
 				tags[tag] = value
 			# no more sub tags (and no possible composite), add this tag to done set (just to be faster):
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 5acc98ad..b4616a15 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -252,6 +252,16 @@ class CommandAction(ActionBase):
 		# set:
 		self.__dict__[name] = value
 
+	def __delattr__(self, name):
+		if not name.startswith('_'):
+			# parameters changed - clear properties and substitution cache:
+			self.__properties = None
+			self.__substCache.clear()
+			#self._logSys.debug("Unset action %r %s", self._name, name)
+			self._logSys.debug("  Unset %s", name)
+		# del:
+		del self.__dict__[name]
+
 	@property
 	def _properties(self):
 		"""A dictionary of the actions properties.
@@ -404,15 +414,19 @@ class CommandAction(ActionBase):
 		# use cache if allowed:
 		if cache is not None:
 			ckey = (query, conditional)
-			value = cache.get(ckey)
-			if value is not None:
-				return value
+			try:
+				return cache[ckey]
+			except KeyError:
+				pass
 
 		# first try get cached tags dictionary:
 		subInfo = csubkey = None
 		if cache is not None:
 			csubkey = ('subst-tags', id(aInfo), conditional)
-			subInfo = cache.get(csubkey)
+			try:
+				subInfo = cache[csubkey]
+			except KeyError:
+				pass
 		# interpolation of dictionary:
 		if subInfo is None:
 			subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
@@ -424,7 +438,7 @@ class CommandAction(ActionBase):
 			cache[csubkey] = subInfo
 
 		# substitution callable, used by interpolation of each tag
-		repeatSubst = {}
+		repeatSubst = {0: 0}
 		def substVal(m):
 			tag = m.group(1)			# tagname from match
 			value = None
@@ -441,19 +455,19 @@ class CommandAction(ActionBase):
 				value = cls.escapeTag(value)
 			# possible contains tags:
 			if '<' in value:
-				repeatSubst[1] = True
+				repeatSubst[0] = 1
 			return value
 
 		# interpolation of query:
 		count = MAX_TAG_REPLACE_COUNT + 1
 		while True:
-			repeatSubst = {}
+			repeatSubst[0] = 0
 			value = TAG_CRE.sub(substVal, query)
 			# possible recursion ?
 			if not repeatSubst or value == query: break
 			query = value
 			count -= 1
-			if count <= 0: # pragma: no cover - almost impossible (because resolved above)
+			if count <= 0:
 				raise ValueError(
 					"unexpected too long replacement interpolation, "
 					"possible self referencing definitions in query: %s" % (query,))
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 38f576b1..435dc1da 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -40,12 +40,20 @@ class CommandActionTest(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
-		self.__action = CommandAction(None, "Test")
 		LogCaptureTestCase.setUp(self)
+		self.__action = CommandAction(None, "Test")
+		# prevent execute stop if start fails (or event not started at all):
+		self.__action_started = False
+		orgstart = self.__action.start
+		def _action_start():
+			self.__action_started = True
+			return orgstart()
+		self.__action.start = _action_start
 
 	def tearDown(self):
 		"""Call after every test case."""
-		self.__action.stop()
+		if self.__action_started:
+			self.__action.stop()
 		LogCaptureTestCase.tearDown(self)
 
 	def testSubstituteRecursiveTags(self):
@@ -196,6 +204,26 @@ class CommandActionTest(LogCaptureTestCase):
 			self.__action.replaceTag("abc",
 				CallingMap(matches=lambda: int("a"))), "abc")
 
+	def testReplaceTagSelfRecursion(self):
+		setattr(self.__action, 'a', "<a")
+		setattr(self.__action, 'b', "c>")
+		setattr(self.__action, 'b?family=inet6', "b>")
+		setattr(self.__action, 'ac', "<a><b>")
+		setattr(self.__action, 'ab', "<ac>")
+		setattr(self.__action, 'x?family=inet6', "")
+		# produce self-referencing properties except:
+		self.assertRaisesRegexp(ValueError, r"properties contain self referencing definitions",
+			lambda: self.__action.replaceTag("<a><b>", 
+				self.__action._properties, conditional="family=inet4")
+		)
+		# remore self-referencing in props:
+		delattr(self.__action, 'ac')
+		# produce self-referencing query except:
+		self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query",
+			lambda: self.__action.replaceTag("<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x>>>>>>>>>>>>>>>>>>>>>", 
+				self.__action._properties, conditional="family=inet6")
+		)
+
 	def testReplaceTagConditionalCached(self):
 		setattr(self.__action, 'abc', "123")
 		setattr(self.__action, 'abc?family=inet4', "345")

From 6d878f3a43854ac30fe6759208cd6b0c7e1c30af Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 16 Feb 2017 22:50:36 +0100
Subject: [PATCH 302/623] try to provide filter captures (already in ticket
 data) to the actions as interpolation options (closes gh-1110)

---
 fail2ban/server/actions.py   |   4 +-
 fail2ban/server/failregex.py | 126 ++++++++++++++++++++++++++---------
 2 files changed, 98 insertions(+), 32 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index e2046e15..3a5eada9 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -350,7 +350,9 @@ class Actions(JailThread, Mapping):
 				"time"		:	bTicket.getTime(),
 				"matches"	:	"\n".join(bTicket.getMatches()),
 				# to bypass actions, that should not be executed for restored tickets
-				"restored":	(1 if ticket.restored else 0)
+				"restored":	(1 if ticket.restored else 0),
+				# extra-interpolation - all match-tags (captured from the filter):
+				"F-*": lambda tag=None: bTicket.getData(tag)
 			})
 			if self._jail.database is not None:
 				mi4ip = lambda overalljails=False, self=self, \
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 7c51ddb8..096cb57a 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -27,6 +27,62 @@ import sys
 
 from .ipdns import IPAddr
 
+
+FTAG_CRE = re.compile(r'</?[\w\-]+/?>')
+
+FCUSTAG_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only
+
+R_HOST = [
+		# separated ipv4:
+		r"""(?:::f{4,6}:)?(?P<ip4>%s)""" % (IPAddr.IP_4_RE,),
+		# separated ipv6:
+		r"""(?P<ip6>%s)""" % (IPAddr.IP_6_RE,),
+		# place-holder for ipv6 enclosed in optional [] (used in addr-, host-regex)
+		"",
+		# separated dns:
+		r"""(?P<dns>[\w\-.^_]*\w)""",
+		# place-holder for ADDR tag-replacement (joined):
+		"",
+		# place-holder for HOST tag replacement (joined):
+		""
+]
+RI_IPV4 =		0
+RI_IPV6 =		1
+RI_IPV6BR =	2
+RI_DNS =		3
+RI_ADDR =		4
+RI_HOST =		5
+
+R_HOST[RI_IPV6BR] =	r"""\[?%s\]?""" % (R_HOST[RI_IPV6],)
+R_HOST[RI_ADDR] =		"(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR])),)
+R_HOST[RI_HOST] =		"(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR], R_HOST[RI_DNS])),)
+
+RH4TAG = {
+	# separated ipv4 (self closed, closed):
+	"IP4":			R_HOST[RI_IPV4],
+	"F-IP4/":		R_HOST[RI_IPV4],
+	# separated ipv6 (self closed, closed):
+	"IP6":			R_HOST[RI_IPV6],
+	"F-IP6/":		R_HOST[RI_IPV6],
+	# 2 address groups instead of <ADDR> - in opposition to `<HOST>`, 
+	# for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together
+	"ADDR":			R_HOST[RI_ADDR],
+	"F-ADDR/":	R_HOST[RI_ADDR],
+	# separated dns (self closed, closed):
+	"DNS":			R_HOST[RI_DNS],
+	"F-DNS/":		R_HOST[RI_DNS],
+	# default failure-id as no space tag:
+	"F-ID/":		r"""(?P<fid>\S+)""",
+	# default failure port, like 80 or http :
+	"F-PORT/": 	r"""(?P<fport>\w+)""",
+}
+
+# default failure groups map for customizable expressions (with different group-id):
+R_MAP = {
+	"ID": "fid",
+	"PORT": "fport",
+}
+
 ##
 # Regular expression class.
 #
@@ -71,38 +127,46 @@ class Regex:
 
 	@staticmethod
 	def _resolveHostTag(regex, useDns="yes"):
-		# separated ipv4:
-		r_host = []
-		r = r"""(?:::f{4,6}:)?(?P<ip4>%s)""" % (IPAddr.IP_4_RE,)
-		regex = regex.replace("<IP4>", r); # self closed
-		regex = regex.replace("<F-IP4/>", r); # closed
-		r_host.append(r)
-		# separated ipv6:
-		r = r"""(?P<ip6>%s)""" % (IPAddr.IP_6_RE,)
-		regex = regex.replace("<IP6>", r); # self closed
-		regex = regex.replace("<F-IP6/>", r); # closed
-		r_host.append(r"""\[?%s\]?""" % (r,)); # enclose ipv6 in optional [] in host-regex
-		# 2 address groups instead of <ADDR> - in opposition to `<HOST>`, 
-		# for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together
-		regex = regex.replace("<ADDR>", "(?:%s)" % ("|".join(r_host),))
-		# separated dns:
-		r = r"""(?P<dns>[\w\-.^_]*\w)"""
-		regex = regex.replace("<DNS>", r); # self closed
-		regex = regex.replace("<F-DNS/>", r); # closed
-		if useDns not in ("no",):
-			r_host.append(r)
-		# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
-		regex = regex.replace("<HOST>", "(?:%s)" % ("|".join(r_host),))
-		# default failure-id as no space tag:
-		regex = regex.replace("<F-ID/>", r"""(?P<fid>\S+)"""); # closed
-		# default failure port, like 80 or http :
-		regex = regex.replace("<F-PORT/>", r"""(?P<port>\w+)"""); # closed
-		# default failure groups (begin / end tag) for customizable expressions:
-		for o,r in (('IP4', 'ip4'), ('IP6', 'ip6'), ('DNS', 'dns'), ('ID', 'fid'), ('PORT', 'fport')):
-			regex = regex.replace("<F-%s>" % o, "(?P<%s>" % r); # open tag
-			regex = regex.replace("</F-%s>" % o, ")"); # close tag
 
-		return regex
+		openTags = dict()
+		# tag interpolation callable:
+		def substTag(m):
+			tag = m.group()
+			tn = tag[1:-1]
+			# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
+			if tn == "HOST":
+				return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR]
+			# static replacement from RH4TAG:
+			try:
+				return RH4TAG[tn]
+			except KeyError:
+				pass
+
+			# (begin / end tag) for customizable expressions, additionally used as
+			# user custom tags (match will be stored in ticket data, can be used in actions):
+			m = FCUSTAG_CRE.match(tn)
+			if m: # match F-...
+				m = m.groups()
+				tn = m[1]
+				# close tag:
+				if m[0]:
+					# check it was already open:
+					if openTags.get(tn):
+						return ")"
+					return tag; # tag not opened, use original
+				# open tag:
+				openTags[tn] = 1
+				try: # if should be mapped:
+					tn = R_MAP[tn]
+				except KeyError:
+					tn = tn.lower()
+				return "(?P<%s>" % (tn,)
+
+			# original, no replacement:
+			return tag
+		
+		# substitute tags:
+		return FTAG_CRE.sub(substTag, regex)
 
 	##
 	# Gets the regular expression.

From 61c8cd11b8f4b553ab014b5581bcb11ddb451e57 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Sat, 18 Feb 2017 00:08:35 +0100
Subject: [PATCH 303/623] Exposes filter group captures in actions
 (non-recursive interpolation of tags `<F-...>`); Closes gh-1110

---
 fail2ban/server/action.py                | 24 +++++++++++++++++++++---
 fail2ban/server/failregex.py             | 16 ++++++++++------
 fail2ban/server/ticket.py                |  4 +++-
 fail2ban/tests/actiontestcase.py         | 19 +++++++++++++++----
 fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++---------
 5 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index b4616a15..bf291690 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -23,6 +23,7 @@ __license__ = "GPL"
 
 import logging
 import os
+import re
 import signal
 import subprocess
 import tempfile
@@ -31,6 +32,7 @@ import time
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .failregex import mapTag2Opt
 from .ipdns import asip
 from .mytime import MyTime
 from .utils import Utils
@@ -45,6 +47,10 @@ _cmd_lock = threading.Lock()
 # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
 allowed_ipv6 = True
 
+# capture groups from filter for map to ticket data:
+FCUSTAG_CRE = re.compile(r'<F-([A-Z0-9_\-]+)>'); # currently uppercase only
+
+
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -72,9 +78,9 @@ class CallingMap(MutableMapping):
 	def __getitem__(self, key):
 		value = self.data[key]
 		if callable(value):
-			return value()
-		else:
-			return value
+			value = value()
+			self.data[key] = value
+		return value
 
 	def __setitem__(self, key, value):
 		self.data[key] = value
@@ -546,6 +552,18 @@ class CommandAction(ActionBase):
 		# Replace dynamical tags (don't use cache here)
 		if aInfo is not None:
 			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
+			# Replace ticket options (filter capture groups) non-recursive:
+			if '<' in realCmd:
+				tickData = aInfo.get("F-*")
+				if not tickData: tickData = {}
+				def substTag(m):
+					tn = mapTag2Opt(m.groups()[0])
+					try:
+						return str(tickData[tn])
+					except KeyError:
+						return ""
+				
+				realCmd = FCUSTAG_CRE.sub(substTag, realCmd)
 		else:
 			realCmd = cmd
 
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 096cb57a..cbd0cef0 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -30,7 +30,7 @@ from .ipdns import IPAddr
 
 FTAG_CRE = re.compile(r'</?[\w\-]+/?>')
 
-FCUSTAG_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only
+FCUSTNAME_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only
 
 R_HOST = [
 		# separated ipv4:
@@ -83,6 +83,12 @@ R_MAP = {
 	"PORT": "fport",
 }
 
+def mapTag2Opt(tag):
+	try: # if should be mapped:
+		return R_MAP[tag]
+	except KeyError:
+		return tag.lower()
+
 ##
 # Regular expression class.
 #
@@ -144,7 +150,7 @@ class Regex:
 
 			# (begin / end tag) for customizable expressions, additionally used as
 			# user custom tags (match will be stored in ticket data, can be used in actions):
-			m = FCUSTAG_CRE.match(tn)
+			m = FCUSTNAME_CRE.match(tn)
 			if m: # match F-...
 				m = m.groups()
 				tn = m[1]
@@ -156,10 +162,8 @@ class Regex:
 					return tag; # tag not opened, use original
 				# open tag:
 				openTags[tn] = 1
-				try: # if should be mapped:
-					tn = R_MAP[tn]
-				except KeyError:
-					tn = tn.lower()
+				# if should be mapped:
+				tn = mapTag2Opt(tn)
 				return "(?P<%s>" % (tn,)
 
 			# original, no replacement:
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index c7bb4d47..a66b64ac 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -56,7 +56,9 @@ class Ticket(object):
 		self._time = time if time is not None else MyTime.time()
 		self._data = {'matches': matches or [], 'failures': 0}
 		if data is not None:
-			self._data.update(data)
+			for k,v in data.iteritems():
+				if v is not None:
+					self._data[k] = v
 		if ticket:
 			# ticket available - copy whole information from ticket:
 			self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__)
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 435dc1da..ca908b65 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -329,13 +329,24 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertEqual(self.__action.ROST,"192.0.2.0")
 
 	def testExecuteActionUnbanAinfo(self):
-		aInfo = {
+		aInfo = CallingMap({
 			'ABC': "123",
-		}
-		self.__action.actionban = "touch /tmp/fail2ban.test.123"
-		self.__action.actionunban = "rm /tmp/fail2ban.test.<ABC>"
+			'ip': '192.0.2.1',
+			'F-*': lambda: {
+		  	'fid': 111,
+		  	'fport': 222,
+				'user': "tester"
+			}
+		})
+		self.__action.actionban = "touch /tmp/fail2ban.test.123; echo 'failure <F-ID> of <F-USER> -<F-TEST>- from <ip>:<F-PORT>'"
+		self.__action.actionunban = "rm /tmp/fail2ban.test.<ABC>; echo 'user <F-USER> unbanned'"
 		self.__action.ban(aInfo)
 		self.__action.unban(aInfo)
+		self.assertLogged(
+			" -- stdout: 'failure 111 of tester -- from 192.0.2.1:222'",
+			" -- stdout: 'user tester unbanned'",
+			all=True
+		)
 
 	def testExecuteActionStartEmpty(self):
 		self.__action.actionstart = ""
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index b8417be5..adaf4719 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -761,9 +761,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"[Definition]",
 				"norestored = %(_exec_once)s",
 				"restore = ",
+				"info = ",
 				"actionstart =  echo '[%(name)s] %(actname)s: ** start'", start,
 				"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
-				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s'", ban,
+				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s%(info)s'", ban,
 				"actionunban =  echo '[%(name)s] %(actname)s: -- unban <ip>'", unban,
 				"actionstop =   echo '[%(name)s] %(actname)s: __ stop'", stop,
 			)
@@ -777,28 +778,28 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"usedns = no",
 				"maxretry = 3",
 				"findtime = 10m",
-				"failregex = ^\s*failure (401|403) from <HOST>",
+				"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>",
 				"datepattern = {^LN-BEG}EPOCH",
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",
 				"         test-action1[name='%(__name__)s']" \
 					if 1 in actions else "",
-				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
 				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
 					if 3 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
-				"failregex = ^\s*failure (401|403) from <HOST>",
-				"            ^\s*error (401|403) from <HOST>" \
+				"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>",
+				"            ^\s*error <F-ERRCODE>401|403</F-ERRCODE> from <HOST>" \
 					if 2 in enabled else "",
 				"enabled = true" if 1 in enabled else "",
 				"",
 				"[test-jail2]", "backend = " + backend, "filter =", 
 				"action = ",
-				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
 				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
 					if 3 in actions else "",
@@ -837,7 +838,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
 		# test restored is 0 (both actions available):
 		self.assertLogged(
-			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
+			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0, err-code: 401'",
 			"stdout: '[test-jail1] test-action3: ++ ban 192.0.2.1 restored: 0'",
 			all=True, wait=MID_WAITTIME)
 
@@ -958,8 +959,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		)
 		# test restored is 1 (only test-action2):
 		self.assertLogged(
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1, err-code: 401'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1, err-code: 401'",
 			all=True, wait=MID_WAITTIME)
 		# test test-action3 not executed at all (norestored check):
 		self.assertNotLogged(

From fe06ffca71e9054b21b93237c40c0c53478a19df Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 20 Feb 2017 11:45:03 +0100
Subject: [PATCH 304/623] Fix retrieving of IPv6 address with dnsToIp on some
 systems (default returns AF_INET family only), fix network test-cases.

---
 fail2ban/server/ipdns.py         | 31 ++++++++++++++++++++-----------
 fail2ban/tests/filtertestcase.py |  5 +++++
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index a5625629..757cceba 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -64,16 +64,19 @@ class DNSUtils:
 		if ips is not None: 
 			return ips
 		# retrieve ips
-		try:
-			ips = list()
-			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
-				ip = IPAddr(result[4][0])
-				if ip.isValid:
-					ips.append(ip)
-		except socket.error as e:
-			# todo: make configurable the expired time of cache entry:
-			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			ips = list()
+		ips = list()
+		saveerr = None
+		for fam, ipfam in ((socket.AF_INET, IPAddr.FAM_IPv4), (socket.AF_INET6, IPAddr.FAM_IPv6)):
+			try:
+				for result in socket.getaddrinfo(dns, None, fam, 0, socket.IPPROTO_TCP):
+					ip = IPAddr(result[4][0], ipfam)
+					if ip.isValid:
+						ips.append(ip)
+			except socket.error as e:
+				saveerr = e
+		if not ips and saveerr:
+			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, saveerr)
+
 		DNSUtils.CACHE_nameToIp.set(dns, ips)
 		return ips
 
@@ -140,6 +143,8 @@ class IPAddr(object):
 
 	CIDR_RAW = -2
 	CIDR_UNSPEC = -1
+	FAM_IPv4 = CIDR_RAW - socket.AF_INET
+	FAM_IPv6 = CIDR_RAW - socket.AF_INET6
 
 	def __new__(cls, ipstr, cidr=CIDR_UNSPEC):
 		# check already cached as IPAddr
@@ -191,7 +196,11 @@ class IPAddr(object):
 		self._raw = ipstr
 		# if not raw - recognize family, set addr, etc.:
 		if cidr != IPAddr.CIDR_RAW:
-			for family in [socket.AF_INET, socket.AF_INET6]:
+			if cidr is not None and cidr < IPAddr.CIDR_RAW:
+				family = [IPAddr.CIDR_RAW - cidr]
+			else:
+				family = [socket.AF_INET, socket.AF_INET6]
+			for family in family:
 				try:
 					binary = socket.inet_pton(family, ipstr)
 					self._family = family
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 986cf1f0..174152b9 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -337,6 +337,11 @@ class IgnoreIP(LogCaptureTestCase):
 		for ip in ipList:
 			self.filter.addIgnoreIP(ip)
 			self.assertFalse(self.filter.inIgnoreIPList(ip))
+		if not unittest.F2B.no_network: # pragma: no cover
+			self.assertLogged(
+				'Unable to find a corresponding IP address for 999.999.999.999',
+				'Unable to find a corresponding IP address for abcdef.abcdef',
+				'Unable to find a corresponding IP address for 192.168.0.', all=True)
 
 	def testIgnoreIPCIDR(self):
 		self.filter.addIgnoreIP('192.168.1.0/25')

From 9d15a792a5f0fd818b2723733071bff4e3ab6d90 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 20 Feb 2017 16:46:02 +0100
Subject: [PATCH 305/623] amend to fe06ffca71e9054b21b93237c40c0c53478a19df:
 small optimization using already known IP family

---
 fail2ban/server/filter.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 0540dedc..b7c78b9b 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -564,7 +564,7 @@ class Filter(JailThread):
 			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
 
 			if dateTimeMatch is None:
-				logSys.error("findFailure failed to parse timeText: " + timeText)
+				logSys.error("findFailure failed to parse timeText: %s", timeText)
 				date = self.__lastDate
 
 			else:
@@ -621,10 +621,16 @@ class Filter(JailThread):
 						# failure-id:
 						fid = fail.get('fid')
 						# ip-address or host:
-						host = fail.get('ip4') or fail.get('ip6')
+						host = fail.get('ip4')
 						if host is not None:
+							cidr = IPAddr.FAM_IPv4
 							raw = True
 						else:
+							host = fail.get('ip6')
+							if host is not None:
+								cidr = IPAddr.FAM_IPv6
+								raw = True
+						if host is None:
 							host = fail.get('dns')
 							if host is None:
 								# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
@@ -639,18 +645,14 @@ class Filter(JailThread):
 							# check host equal failure-id, if not - failure with complex id:
 							if fid is not None and fid != host:
 								ip = IPAddr(fid, IPAddr.CIDR_RAW)
-							failList.append([failRegexIndex, ip, date,
-								failRegex.getMatchedLines(), fail])
-							if not self.checkAllRegex:
-								break
+							ips = [ip]
 						else:
 							ips = DNSUtils.textToIp(host, self.__useDns)
-							if ips:
-								for ip in ips:
-									failList.append([failRegexIndex, ip, date,
-										failRegex.getMatchedLines(), fail])
-								if not self.checkAllRegex:
-									break
+						for ip in ips:
+							failList.append([failRegexIndex, ip, date,
+								failRegex.getMatchedLines(), fail])
+						if not self.checkAllRegex:
+							break
 					except RegexException as e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
 		return failList

From 4ff8d051f49808ac769709c5aff8591fcd79040a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 20 Feb 2017 16:42:51 +0100
Subject: [PATCH 306/623] Introduced new filter option `prefregex` for
 pre-filtering using single regular expression; Some filters extended with
 user name; [filter.d/pam-generic.conf]: grave fix injection on user name to
 host fixed; test-cases in testSampleRegexsFactory can now check the captured
 groups (using additionally fields in failJSON structure)

---
 config/filter.d/pam-generic.conf      |  7 ++-
 config/filter.d/sshd.conf             | 30 +++++-----
 fail2ban/client/fail2banregex.py      | 74 +++++++++++++-----------
 fail2ban/client/filterreader.py       |  5 +-
 fail2ban/server/failregex.py          | 81 ++++++++++++++-------------
 fail2ban/server/filter.py             | 43 ++++++++++++--
 fail2ban/server/server.py             |  8 +++
 fail2ban/server/transmitter.py        |  6 ++
 fail2ban/tests/files/logs/pam-generic | 14 +++--
 fail2ban/tests/samplestestcase.py     | 18 ++++--
 10 files changed, 183 insertions(+), 103 deletions(-)

diff --git a/config/filter.d/pam-generic.conf b/config/filter.d/pam-generic.conf
index e0d4e9c1..ff4ea802 100644
--- a/config/filter.d/pam-generic.conf
+++ b/config/filter.d/pam-generic.conf
@@ -16,7 +16,12 @@ _ttys_re=\S*
 __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:?
 _daemon = \S+
 
-failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s ruser=\S* rhost=<HOST>(?:\s+user=.*)?\s*$
+prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^ruser=<F-USER>\S*</F-USER> rhost=<HOST>\s*$
+            ^ruser= rhost=<HOST>\s+user=<F-USER>\S*</F-USER>\s*$
+            ^ruser= rhost=<HOST>\s+user=<F-USER>.*?</F-USER>\s*$
+            ^ruser=<F-USER>.*?</F-USER> rhost=<HOST>\s*$
 
 ignoreregex = 
 
diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index ebdc06ec..872a73e4 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -32,23 +32,23 @@ __prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
 
 mode = %(normal)s
 
-normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-         ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__on_port_opt)s\s*$
-         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not in any group\s*%(__suff)s$
+normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)s<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
+         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
          ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)\s*%(__suff)s$
          ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
-         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$
-         ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
-         ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
-         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
-         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$
+         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
+         ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+         ^%(__prefix_line_ml1)sUser <F-USER>.+</F-USER> not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
+         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for <F-USER>.+</F-USER>%(__suff)s$
 
 ddos =   ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
          ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index d111e09c..84106d02 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -271,7 +271,7 @@ class Fail2banRegex(object):
 	def readRegex(self, value, regextype):
 		assert(regextype in ('fail', 'ignore'))
 		regex = regextype + 'regex'
-		if os.path.isfile(value) or os.path.isfile(value + '.conf'):
+		if regextype == 'fail' and (os.path.isfile(value) or os.path.isfile(value + '.conf')):
 			if os.path.basename(os.path.dirname(value)) == 'filter.d':
 				## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.):
 				basedir = os.path.dirname(os.path.dirname(value))
@@ -291,43 +291,51 @@ class Fail2banRegex(object):
 					return False
 			reader.getOptions(None)
 			readercommands = reader.convert()
-			regex_values = [
-				RegexStat(m[3])
-				for m in filter(
-					lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype,
-					readercommands)
-			] + [
-				RegexStat(m)
-				for mm in filter(
-					lambda x: x[0] == 'multi-set' and x[2] == "add%sregex" % regextype,
-					readercommands)
-				for m in mm[3]
-			]
-			# Read out and set possible value of maxlines
-			for command in readercommands:
-				if command[2] == "maxlines":
-					maxlines = int(command[3])
+
+			regex_values = {}
+			for opt in readercommands:
+				if opt[0] == 'multi-set':
+					optval = opt[3]
+				elif opt[0] == 'set':
+					optval = [opt[3]]
+				else:
+					continue
+				for optval in optval:
 					try:
-						self.setMaxLines(maxlines)
-					except ValueError:
-						output( "ERROR: Invalid value for maxlines (%(maxlines)r) " \
-							  "read from %(value)s" % locals() )
+						if opt[2] == "prefregex":
+							self._filter.prefRegex = optval
+						elif opt[2] == "addfailregex":
+							stor = regex_values.get('fail')
+							if not stor: stor = regex_values['fail'] = list()
+							stor.append(RegexStat(optval))
+							#self._filter.addFailRegex(optval)
+						elif opt[2] == "addignoreregex":
+							stor = regex_values.get('ignore')
+							if not stor: stor = regex_values['ignore'] = list()
+							stor.append(RegexStat(optval))
+							#self._filter.addIgnoreRegex(optval)
+						elif opt[2] == "maxlines":
+							self.setMaxLines(optval)
+						elif opt[2] == "datepattern":
+							self.setDatePattern(optval)
+						elif opt[2] == "addjournalmatch":
+							self.setJournalMatch(optval)
+					except ValueError as e: # pragma: no cover
+						output( "ERROR: Invalid value for %s (%r) " \
+							  "read from %s: %s" % (opt[2], optval, value, e) )
 						return False
-				elif command[2] == 'addjournalmatch':
-					journalmatch = command[3:]
-					self.setJournalMatch(journalmatch)
-				elif command[2] == 'datepattern':
-					datepattern = command[3]
-					self.setDatePattern(datepattern)
+
 		else:
 			output( "Use %11s line : %s" % (regex, shortstr(value)) )
-			regex_values = [RegexStat(value)]
+			regex_values = {regextype: [RegexStat(value)]}
 
-		setattr(self, "_" + regex, regex_values)
-		for regex in regex_values:
-			getattr(
-				self._filter,
-				'add%sRegex' % regextype.title())(regex.getFailRegex())
+		for regextype, regex_values in regex_values.iteritems():
+			regex = regextype + 'regex'
+			setattr(self, "_" + regex, regex_values)
+			for regex in regex_values:
+				getattr(
+					self._filter,
+					'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True
 
 	def testIgnoreRegex(self, line):
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index d89ef5ad..59e78307 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -37,6 +37,7 @@ logSys = getLogger(__name__)
 class FilterReader(DefinitionInitConfigReader):
 
 	_configOpts = {
+		"prefregex": ["string", None],
 		"ignoreregex": ["string", None],
 		"failregex": ["string", ""],
 		"maxlines": ["int", None],
@@ -72,8 +73,8 @@ class FilterReader(DefinitionInitConfigReader):
 				# We warn when multiline regex is used without maxlines > 1
 				# therefore keep sure we set this option first.
 				stream.insert(0, ["set", self._jailName, "maxlines", value])
-			elif opt == 'datepattern':
-				stream.append(["set", self._jailName, "datepattern", value])
+			elif opt in ('datepattern', 'prefregex'):
+				stream.append(["set", self._jailName, opt, value])
 			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
 				if value is None: continue
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index cbd0cef0..5f6ca01f 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -189,40 +189,45 @@ class Regex:
 	# method of this object.
 	# @param a list of tupples. The tupples are ( prematch, datematch, postdatematch )
 	
-	def search(self, tupleLines):
+	def search(self, tupleLines, orgLines=None):
 		self._matchCache = self._regexObj.search(
 			"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
-		if self.hasMatched():
-			# Find start of the first line where the match was found
-			try:
-				self._matchLineStart = self._matchCache.string.rindex(
-					"\n", 0, self._matchCache.start() +1 ) + 1
-			except ValueError:
-				self._matchLineStart = 0
-			# Find end of the last line where the match was found
-			try:
-				self._matchLineEnd = self._matchCache.string.index(
-					"\n", self._matchCache.end() - 1) + 1
-			except ValueError:
-				self._matchLineEnd = len(self._matchCache.string)
+		if self._matchCache:
+			if orgLines is None: orgLines = tupleLines
+			# if single-line:
+			if len(orgLines) <= 1:
+				self._matchedTupleLines = orgLines
+				self._unmatchedTupleLines = []
+			else:
+				# Find start of the first line where the match was found
+				try:
+					matchLineStart = self._matchCache.string.rindex(
+						"\n", 0, self._matchCache.start() +1 ) + 1
+				except ValueError:
+					matchLineStart = 0
+				# Find end of the last line where the match was found
+				try:
+					matchLineEnd = self._matchCache.string.index(
+						"\n", self._matchCache.end() - 1) + 1
+				except ValueError:
+					matchLineEnd = len(self._matchCache.string)
 
-			lineCount1 = self._matchCache.string.count(
-				"\n", 0, self._matchLineStart)
-			lineCount2 = self._matchCache.string.count(
-				"\n", 0, self._matchLineEnd)
-			self._matchedTupleLines = tupleLines[lineCount1:lineCount2]
-			self._unmatchedTupleLines = tupleLines[:lineCount1]
-
-			n = 0
-			for skippedLine in self.getSkippedLines():
-				for m, matchedTupleLine in enumerate(
-					self._matchedTupleLines[n:]):
-					if "".join(matchedTupleLine[::2]) == skippedLine:
-						self._unmatchedTupleLines.append(
-							self._matchedTupleLines.pop(n+m))
-						n += m
-						break
-			self._unmatchedTupleLines.extend(tupleLines[lineCount2:])
+				lineCount1 = self._matchCache.string.count(
+					"\n", 0, matchLineStart)
+				lineCount2 = self._matchCache.string.count(
+					"\n", 0, matchLineEnd)
+				self._matchedTupleLines = orgLines[lineCount1:lineCount2]
+				self._unmatchedTupleLines = orgLines[:lineCount1]
+				n = 0
+				for skippedLine in self.getSkippedLines():
+					for m, matchedTupleLine in enumerate(
+						self._matchedTupleLines[n:]):
+						if "".join(matchedTupleLine[::2]) == skippedLine:
+							self._unmatchedTupleLines.append(
+								self._matchedTupleLines.pop(n+m))
+							n += m
+							break
+				self._unmatchedTupleLines.extend(orgLines[lineCount2:])
 
 	# Checks if the previous call to search() matched.
 	#
@@ -234,6 +239,13 @@ class Regex:
 		else:
 			return False
 
+	##
+	# Returns all matched groups.
+	#
+
+	def getGroups(self):
+		return self._matchCache.groupdict()
+
 	##
 	# Returns skipped lines.
 	#
@@ -332,13 +344,6 @@ class FailRegex(Regex):
 		if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]:
 			raise RegexException("No failure-id group in '%s'" % self._regex)
 	
-	##
-	# Returns all matched groups.
-	#
-
-	def getGroups(self):
-		return self._matchCache.groupdict()
-
 	##
 	# Returns the matched failure id.
 	#
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b7c78b9b..d25e8dc5 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -65,6 +65,8 @@ class Filter(JailThread):
 		self.jail = jail
 		## The failures manager.
 		self.failManager = FailManager()
+		## Regular expression pre-filtering matching the failures.
+		self.__prefRegex = None
 		## The regular expression list matching the failures.
 		self.__failRegex = list()
 		## The regular expression list with expressions to ignore.
@@ -129,6 +131,16 @@ class Filter(JailThread):
 					self.delLogPath(path)
 				delattr(self, '_reload_logs')
 
+	@property
+	def prefRegex(self):
+		return self.__prefRegex
+	@prefRegex.setter
+	def prefRegex(self, value):
+		if value:
+			self.__prefRegex = Regex(value, useDns=self.__useDns)
+		else:
+			self.__prefRegex = None
+
 	##
 	# Add a regular expression which matches the failure.
 	#
@@ -582,13 +594,30 @@ class Filter(JailThread):
 				date, MyTime.time(), self.getFindTime())
 			return failList
 
-		self.__lineBuffer = (
-			self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
-		logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer)
+		if self.__lineBufferSize > 1:
+			orgBuffer = self.__lineBuffer = (
+				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
+		else:
+			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
+		logSys.log(5, "Looking for failregex match of %r", self.__lineBuffer)
+
+		# Pre-filter fail regex (if available):
+		preGroups = {}
+		if self.__prefRegex:
+			failRegex = self.__prefRegex.search(self.__lineBuffer)
+			if not self.__prefRegex.hasMatched():
+				return failList
+			logSys.log(7, "Pre-filter matched %s", failRegex)
+			preGroups = self.__prefRegex.getGroups()
+			repl = preGroups.get('content')
+			# Content replacement:
+			if repl:
+				del preGroups['content']
+				self.__lineBuffer = [('', '', repl)]
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
-			failRegex.search(self.__lineBuffer)
+			failRegex.search(self.__lineBuffer, orgBuffer)
 			if failRegex.hasMatched():
 				# The failregex matched.
 				logSys.log(7, "Matched %s", failRegex)
@@ -617,7 +646,11 @@ class Filter(JailThread):
 					# retrieve failure-id, host, etc from failure match:
 					raw = returnRawHost
 					try:
-						fail = failRegex.getGroups()
+						if preGroups:
+							fail = preGroups.copy()
+							fail.update(failRegex.getGroups())
+						else:
+							fail = failRegex.getGroups()
 						# failure-id:
 						fid = fail.get('fid')
 						# ip-address or host:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 313b6ee5..dfab1e38 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -379,6 +379,14 @@ class Server:
 	def getIgnoreCommand(self, name):
 		return self.__jails[name].filter.getIgnoreCommand()
 
+	def setPrefRegex(self, name, value):
+		flt = self.__jails[name].filter
+		logSys.debug("  prefregex: %r", value)
+		flt.prefRegex = value
+
+	def getPrefRegex(self, name):
+		return self.__jails[name].filter.prefRegex
+	
 	def addFailRegex(self, name, value, multiple=False):
 		flt = self.__jails[name].filter
 		if not multiple: value = (value,)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index d23f12e2..265b9704 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -221,6 +221,10 @@ class Transmitter:
 			value = command[2:]
 			self.__server.delJournalMatch(name, value)
 			return self.__server.getJournalMatch(name)
+		elif command[1] == "prefregex":
+			value = command[2]
+			self.__server.setPrefRegex(name, value)
+			return self.__server.getPrefRegex(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value, multiple=multiple)
@@ -341,6 +345,8 @@ class Transmitter:
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "ignorecommand":
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "prefregex":
+			return self.__server.getPrefRegex(name)
 		elif command[1] == "failregex":
 			return self.__server.getFailRegex(name)
 		elif command[1] == "ignoreregex":
diff --git a/fail2ban/tests/files/logs/pam-generic b/fail2ban/tests/files/logs/pam-generic
index e562ac7f..1740f0c8 100644
--- a/fail2ban/tests/files/logs/pam-generic
+++ b/fail2ban/tests/files/logs/pam-generic
@@ -1,17 +1,23 @@
-# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1" }
+# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1", "user": "sample-user" }
 Feb  7 15:10:42 example pure-ftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=pure-ftpd ruser=sample-user rhost=192.168.1.1 
-# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" }
+# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com", "user": "root" }
 May 12 09:47:54 vaio sshd[16004]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com  user=root
 # failJSON: { "time": "2005-05-12T09:48:03", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" }
 May 12 09:48:03 vaio sshd[16021]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com
-# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62" }
+# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62", "user": "mark" }
 May 15 18:02:12 localhost proftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=66.232.129.62  user=mark
 
 # linux-pam messages before commit f0f9c4479303b5a9c37667cf07f58426dc081676 (release 0.99.2.0 ) - nolonger supported
 # failJSON: { "time": "2004-11-25T17:12:13", "match": false }
 Nov 25 17:12:13 webmail pop(pam_unix)[4920]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.168.10.3 user=mailuser
 
-# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com" }
+# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com", "user": "an8767" }
 Jul 19 18:11:26 srv2 vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www.google.com
 # failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com" }
 Jul 19 18:11:26 srv2 vsftpd: pam_unix: authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www.google.com
+
+
+# failJSON: { "time": "2005-07-19T18:11:50", "match": true , "host": "192.0.2.1", "user": "test rhost=192.0.2.151", "desc": "Injecting on username"}
+Jul 19 18:11:50 srv2 daemon: pam_unix(auth): authentication failure; logname= uid=0 euid=0 tty=xxx ruser=test rhost=192.0.2.151 rhost=192.0.2.1
+# failJSON: { "time": "2005-07-19T18:11:52", "match": true , "host": "192.0.2.2", "user": "test rhost=192.0.2.152", "desc": "Injecting on username after host"}
+Jul 19 18:11:52 srv2 daemon: pam_unix(auth): authentication failure; logname= uid=0 euid=0 tty=xxx ruser= rhost=192.0.2.2 user=test rhost=192.0.2.152
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 3fdc50c6..a97e92c3 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -102,7 +102,9 @@ def testSampleRegexsFactory(name, basedir):
 			else:
 				continue
 			for optval in optval:
-				if opt[2] == "addfailregex":
+				if opt[2] == "prefregex":
+					self.filter.prefRegex = optval
+				elif opt[2] == "addfailregex":
 					self.filter.addFailRegex(optval)
 				elif opt[2] == "addignoreregex":
 					self.filter.addIgnoreRegex(optval)
@@ -126,7 +128,7 @@ def testSampleRegexsFactory(name, basedir):
 			# test regexp contains greedy catch-all before <HOST>, that is
 			# not hard-anchored at end or has not precise sub expression after <HOST>:
 			for fr in self.filter.getFailRegex():
-				if RE_WRONG_GREED.search(fr): #pragma: no cover
+				if RE_WRONG_GREED.search(fr): # pragma: no cover
 					raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before <HOST>, "
 						"that is not hard-anchored at end or has not precise sub expression after <HOST>:\n%s" %
 						(name, str(fr).replace(RE_HOST, '<HOST>')))
@@ -152,12 +154,12 @@ def testSampleRegexsFactory(name, basedir):
 				if not ret:
 					# Check line is flagged as none match
 					self.assertFalse(faildata.get('match', True),
-						 "Line not matched when should have: %s:%i %r" %
+						 "Line not matched when should have: %s:%i, line:\n%s" %
 						(logFile.filename(), logFile.filelineno(), line))
 				elif ret:
 					# Check line is flagged to match
 					self.assertTrue(faildata.get('match', False),
-						"Line matched when shouldn't have: %s:%i %r" %
+						"Line matched when shouldn't have: %s:%i, line:\n%s" %
 						(logFile.filename(), logFile.filelineno(), line))
 					self.assertEqual(len(ret), 1, "Multiple regexs matched %r - %s:%i" %
 									 (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
@@ -165,6 +167,12 @@ def testSampleRegexsFactory(name, basedir):
 					# Verify timestamp and host as expected
 					failregex, host, fail2banTime, lines, fail = ret[0]
 					self.assertEqual(host, faildata.get("host", None))
+					# Verify other captures:
+					for k, v in faildata.iteritems():
+						if k not in ("time", "match", "host", "desc"):
+							fv = fail.get(k, None)
+							self.assertEqual(fv, v, "Value of %s mismatch %r != %r on: %s:%i, line:\n%s" % (
+								k, fv, v, logFile.filename(), logFile.filelineno(), line))
 
 					t = faildata.get("time", None)
 					try:
@@ -177,7 +185,7 @@ def testSampleRegexsFactory(name, basedir):
 					jsonTime += jsonTimeLocal.microsecond / 1000000
 
 					self.assertEqual(fail2banTime, jsonTime,
-						"UTC Time  mismatch fail2ban %s (%s) != failJson %s (%s)  (diff %.3f seconds) on: %s:%i %r:" % 
+						"UTC Time  mismatch %s (%s) != %s (%s)  (diff %.3f seconds) on: %s:%i, line:\n%s" % 
 						(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
 						jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
 						fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) )

From 2fad50b6e8296a03fecc12b542d62350ad662ff5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Feb 2017 15:54:25 +0100
Subject: [PATCH 307/623] Precedence of `prefregex` higher as `failregex`
 should be in head of the convert-stream; Allow using failure-id (`<HOST>`)
 within `prefregex` (by common prefix for all expressions specified with
 `failregex`)

---
 fail2ban/client/filterreader.py | 9 ++++-----
 fail2ban/server/failregex.py    | 7 +++++--
 fail2ban/server/filter.py       | 6 +++---
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index 59e78307..9edeb2f3 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -69,11 +69,10 @@ class FilterReader(DefinitionInitConfigReader):
 					stream.append(["multi-set", self._jailName, "add" + opt, multi])
 				elif len(multi):
 					stream.append(["set", self._jailName, "add" + opt, multi[0]])
-			elif opt == 'maxlines':
-				# We warn when multiline regex is used without maxlines > 1
-				# therefore keep sure we set this option first.
-				stream.insert(0, ["set", self._jailName, "maxlines", value])
-			elif opt in ('datepattern', 'prefregex'):
+			elif opt in ('maxlines', 'prefregex'):
+				# Be sure we set this options first.
+				stream.insert(0, ["set", self._jailName, opt, value])
+			elif opt in ('datepattern'):
 				stream.append(["set", self._jailName, opt, value])
 			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 5f6ca01f..19a89303 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -337,11 +337,14 @@ class FailRegex(Regex):
 	# avoid construction of invalid object.
 	# @param value the regular expression
 
-	def __init__(self, regex, **kwargs):
+	def __init__(self, regex, prefRegex=None, **kwargs):
 		# Initializes the parent.
 		Regex.__init__(self, regex, **kwargs)
 		# Check for group "dns", "ip4", "ip6", "fid"
-		if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]:
+		if (not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]
+			and (prefRegex is None or
+				not [grp for grp in FAILURE_ID_GROPS if grp in prefRegex._regexObj.groupindex])
+		):
 			raise RegexException("No failure-id group in '%s'" % self._regex)
 	
 	##
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index d25e8dc5..85c07103 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -150,7 +150,7 @@ class Filter(JailThread):
 
 	def addFailRegex(self, value):
 		try:
-			regex = FailRegex(value, useDns=self.__useDns)
+			regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns)
 			self.__failRegex.append(regex)
 			if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
 				logSys.warning(
@@ -604,11 +604,11 @@ class Filter(JailThread):
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
-			failRegex = self.__prefRegex.search(self.__lineBuffer)
+			self.__prefRegex.search(self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
 				return failList
-			logSys.log(7, "Pre-filter matched %s", failRegex)
 			preGroups = self.__prefRegex.getGroups()
+			logSys.log(7, "Pre-filter matched %s", preGroups)
 			repl = preGroups.get('content')
 			# Content replacement:
 			if repl:

From 22afdbd536a193fe8ee7ff8dcc0ed7dd6cd23342 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Feb 2017 15:54:59 +0100
Subject: [PATCH 308/623] Several filters optimized with pre-filtering using
 new option `prefregex`

---
 config/filter.d/apache-auth.conf       | 32 +++++++++++++++-----------
 config/filter.d/apache-botsearch.conf  |  9 ++++----
 config/filter.d/apache-shellshock.conf |  6 +++--
 config/filter.d/asterisk.conf          | 22 ++++++++++--------
 config/filter.d/courier-smtp.conf      |  6 +++--
 config/filter.d/dovecot.conf           | 15 +++++++-----
 config/filter.d/dropbear.conf          |  8 ++++---
 config/filter.d/exim-common.conf       |  4 +++-
 config/filter.d/exim.conf              |  7 ++++--
 config/filter.d/froxlor-auth.conf      |  7 ++++--
 config/filter.d/murmur.conf            |  6 +++--
 config/filter.d/named-refused.conf     |  8 ++++---
 config/filter.d/postfix.conf           | 16 +++++++------
 config/filter.d/proftpd.conf           | 12 ++++++----
 config/filter.d/xinetd-fail.conf       |  6 +++--
 15 files changed, 99 insertions(+), 65 deletions(-)

diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf
index 8a63858d..c59a817c 100644
--- a/config/filter.d/apache-auth.conf
+++ b/config/filter.d/apache-auth.conf
@@ -9,20 +9,24 @@ before = apache-common.conf
 
 [Definition]
 
+prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$
-            ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$
-            ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*?: password mismatch: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01631: )?user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01793: )?invalid qop `.*?' received: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$
+# auth_type = ((?:Digest|Basic): )?
+auth_type = ([A-Z]\w+: )?
+
+failregex = ^client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
+            ^user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$
+            ^user .*? not found(: )?\S*(, referer: \S+)?\s*$
+            ^client used wrong authentication scheme: \S*(, referer: \S+)?\s*$
+            ^Authorization of user \S+ to access \S* failed, reason: .*$
+            ^%(auth_type)suser .*?: password mismatch: \S*(, referer: \S+)?\s*$
+            ^%(auth_type)suser `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$
+            ^user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$
+            ^%(auth_type)sinvalid nonce .* received - length is not \S+(, referer: \S+)?\s*$
+            ^%(auth_type)srealm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$
+            ^%(auth_type)sunknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$
+            ^invalid qop `.*?' received: \S*(, referer: \S+)?\s*$
+            ^%(auth_type)sinvalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$
 
 ignoreregex = 
 
@@ -53,4 +57,4 @@ ignoreregex =
 # referer is always in error log messages if it exists added as per the log_error_core function in server/log.c
 # 
 # Author: Cyril Jaquier
-# Major edits by Daniel Black
+# Major edits by Daniel Black and Sergey Brester (sebres)
diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf
index 5687d405..7def09ac 100644
--- a/config/filter.d/apache-botsearch.conf
+++ b/config/filter.d/apache-botsearch.conf
@@ -23,14 +23,13 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): <webroot><block>(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s script '<webroot><block>' not found or unable to stat(, referer: \S+)?\s*$
+prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(?:File does not exist|script not found or unable to stat): <webroot><block>(, referer: \S+)?\s*$
+            ^script '<webroot><block>' not found or unable to stat(, referer: \S+)?\s*$
 
 ignoreregex = 
 
-
-[Init]
-
 # Webroot represents the webroot on which all other files are based
 webroot = /var/www/
 
diff --git a/config/filter.d/apache-shellshock.conf b/config/filter.d/apache-shellshock.conf
index 39df1704..e2707dc0 100644
--- a/config/filter.d/apache-shellshock.conf
+++ b/config/filter.d/apache-shellshock.conf
@@ -9,8 +9,10 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: warning: HTTP_.*?: ignoring function definition attempt(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: error importing function definition for `HTTP_.*?'(, referer: \S+)?\s*$
+prefregex = ^%(_apache_error_client)s (AH01215: )?/bin/([bd]a)?sh: <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^warning: HTTP_[^:]+: ignoring function definition attempt(, referer: \S+)?\s*$
+            ^error importing function definition for `HTTP_[^']+'(, referer: \S+)?\s*$
 
 ignoreregex = 
 
diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index df55d288..b9871b6c 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -18,16 +18,18 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 # All Asterisk log messages begin like this:
 log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)?
 
-failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
-            ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
-            ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
-            ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
-            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
+prefregex = ^%(__prefix_line)s%(log_prefix)s <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+            ^Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^Host <HOST> failed to authenticate as '[^']*'$
+            ^No registration for peer '[^']*' \(from <HOST>\)$
+            ^Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^Failed to authenticate (user|device) [^@]+@<HOST>\S*$
+            ^hacking attempt detected '<HOST>'$
+            ^SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
+            ^"Rejecting unknown SIP connection from <HOST>"$
+            ^Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 
 ignoreregex =
 
diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf
index fc0afc26..888753c4 100644
--- a/config/filter.d/courier-smtp.conf
+++ b/config/filter.d/courier-smtp.conf
@@ -12,8 +12,10 @@ before = common.conf
 
 _daemon = courieresmtpd
 
-failregex = ^%(__prefix_line)serror,relay=<HOST>,.*: 550 User (<.*> )?unknown\.?$
-            ^%(__prefix_line)serror,relay=<HOST>,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$
+prefregex = ^%(__prefix_line)serror,relay=<HOST>,<F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$
+            ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$
 
 ignoreregex = 
 
diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index 3c45a332..966a7e86 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -7,13 +7,16 @@ before = common.conf
 
 [Definition]
 
-_daemon = (auth|dovecot(-auth)?|auth-worker)
+_auth_worker = (?:dovecot: )?auth(?:-worker)?
+_daemon = (dovecot(-auth)?|auth)
 
-failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
-            ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
-            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
-            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$
+prefregex = ^%(__prefix_line)s(%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
+            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
+            ^pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
+            ^(?:pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
+            ^ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$
 
 ignoreregex = 
 
diff --git a/config/filter.d/dropbear.conf b/config/filter.d/dropbear.conf
index 288b0882..930bb128 100644
--- a/config/filter.d/dropbear.conf
+++ b/config/filter.d/dropbear.conf
@@ -23,9 +23,11 @@ before = common.conf
 
 _daemon = dropbear
 
-failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from <HOST>:\d+$
-            ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from <HOST>(:\d+)?$
-            ^%(__prefix_line)s[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from <HOST>:\d+\s*$
+prefregex = ^%(__prefix_line)s<F-CONTENT>(?:[Ll]ogin|[Bb]ad|[Ee]xit).+</F-CONTENT>$
+
+failregex = ^[Ll]ogin attempt for nonexistent user ('.*' )?from <HOST>:\d+$
+            ^[Bb]ad (PAM )?password attempt for .+ from <HOST>(:\d+)?$
+            ^[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from <HOST>:\d+\s*$
 
 ignoreregex = 
 
diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf
index 0e1b74fa..b3b25750 100644
--- a/config/filter.d/exim-common.conf
+++ b/config/filter.d/exim-common.conf
@@ -9,7 +9,9 @@ after = exim-common.local
 
 [Definition]
 
-host_info = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?\[<HOST>\](?::\d+)? (?:I=\[\S+\](:\d+)? )?(?:U=\S+ )?(?:P=e?smtp )?
+host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?
+host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s
+host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s
 pid = (?: \[\d+\])?
 
 # DEV Notes:
diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index a1d699c0..5d293429 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -13,14 +13,17 @@ before = exim-common.conf
 
 [Definition]
 
+# Fre-filter via "prefregex" is currently inactive because of too different failure syntax in exim-log (testing needed):
+#prefregex = ^%(pid)s <F-CONTENT>\b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info_pre)s\[[^\]]+\]%(host_info_suf)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+</F-CONTENT>$
+
 failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
             ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[<HOST>\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$
+            ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
             ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$
-            ^%(pid)s \S+ SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
+            ^%(pid)s ([\w\-]+ )?SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 
 ignoreregex = 
 
diff --git a/config/filter.d/froxlor-auth.conf b/config/filter.d/froxlor-auth.conf
index 04003263..d8f3785c 100644
--- a/config/filter.d/froxlor-auth.conf
+++ b/config/filter.d/froxlor-auth.conf
@@ -25,8 +25,11 @@ _daemon = Froxlor
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = ^%(__prefix_line)s\[Login Action <HOST>\] Unknown user \S* tried to login.$
-            ^%(__prefix_line)s\[Login Action <HOST>\] User \S* tried to login with wrong password.$
+
+prefregex = ^%(__prefix_line)s\[Login Action <HOST>\] <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^Unknown user \S* tried to login.$
+            ^User \S* tried to login with wrong password.$
 
 
 # Option:  ignoreregex
diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf
index 507bbd2f..f5f100a6 100644
--- a/config/filter.d/murmur.conf
+++ b/config/filter.d/murmur.conf
@@ -17,8 +17,10 @@ _usernameregex = [^>]+
 
 _prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
 
-failregex = ^%(_prefix)s Invalid server password$
-            ^%(_prefix)s Wrong certificate or password for existing user$
+prefregex = ^%(_prefix)s <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^Invalid server password$
+            ^Wrong certificate or password for existing user$
 
 ignoreregex =
 
diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf
index eec3d667..2e14d442 100644
--- a/config/filter.d/named-refused.conf
+++ b/config/filter.d/named-refused.conf
@@ -34,9 +34,11 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)
 # this can be optional (for instance if we match named native log files)
 __line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)?
 
-failregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
-            ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
-            ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
+prefregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
+            ^zone transfer '\S+/AXFR/\w+' denied\s*$
+            ^bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
 
 ignoreregex =
 
diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index 3051409b..005ed585 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -12,13 +12,15 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 554 5\.7\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
-            ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
-            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]: 550 5\.1\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[<HOST>\]:?$
+prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^RCPT from \S+\[<HOST>\]: 554 5\.7\.1 
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+            ^EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
+            ^VRFY from \S+\[<HOST>\]: 550 5\.1\.1 
+            ^RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^after \S+ from [^[]*\[<HOST>\]:?$
 
 ignoreregex = 
 
diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf
index 4bc0ba01..455bbdfc 100644
--- a/config/filter.d/proftpd.conf
+++ b/config/filter.d/proftpd.conf
@@ -16,10 +16,14 @@ _daemon = proftpd
 
 __suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).?
 
-failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
-            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
-            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$
-            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$
+
+prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum).+</F-CONTENT>$
+
+
+failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
+            ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
+            ^SECURITY VIOLATION: .* login attempted\. *$
+            ^Maximum login attempts \(\d+\) exceeded *$
 
 ignoreregex = 
 
diff --git a/config/filter.d/xinetd-fail.conf b/config/filter.d/xinetd-fail.conf
index d75e3d66..b4093d98 100644
--- a/config/filter.d/xinetd-fail.conf
+++ b/config/filter.d/xinetd-fail.conf
@@ -14,8 +14,10 @@ before = common.conf
 
 _daemon = xinetd
 
-failregex = ^%(__prefix_line)sFAIL: \S+ address from=<HOST>$
-            ^%(__prefix_line)sFAIL: \S+ libwrap from=<HOST>$
+prefregex = ^%(__prefix_line)sFAIL: <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^\S+ address from=<HOST>$
+            ^\S+ libwrap from=<HOST>$
 
 ignoreregex = 
 

From 8bcaeb9022543a31214c1d38881afc0b6f9e6783 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Feb 2017 17:03:36 +0100
Subject: [PATCH 309/623] amend to 4ff8d051f49808ac769709c5aff8591fcd79040a:
 fixed fail2ban-regex with journalmatch using systemd-journal

---
 fail2ban/client/fail2banregex.py        | 49 ++++++++++++++-----------
 fail2ban/tests/fail2banregextestcase.py | 24 ++++++------
 2 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 84106d02..395fe8ef 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -235,7 +235,7 @@ class Fail2banRegex(object):
 		else:
 			self._maxlines = 20
 		if opts.journalmatch is not None:
-			self.setJournalMatch(opts.journalmatch.split())
+			self.setJournalMatch(shlex.split(opts.journalmatch))
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
 		if opts.usedns:
@@ -243,6 +243,7 @@ class Fail2banRegex(object):
 		self._filter.returnRawHost = opts.raw
 		self._filter.checkFindTime = False
 		self._filter.checkAllRegex = True
+		self._opts = opts
 
 	def decode_line(self, line):
 		return FileContainer.decode_line('<LOG>', self._encoding, line)
@@ -265,8 +266,7 @@ class Fail2banRegex(object):
 			output( "Use         maxlines : %d" % self._filter.getMaxLines() )
 
 	def setJournalMatch(self, v):
-		if self._journalmatch is None:
-			self._journalmatch = v
+		self._journalmatch = v
 
 	def readRegex(self, value, regextype):
 		assert(regextype in ('fail', 'ignore'))
@@ -297,33 +297,38 @@ class Fail2banRegex(object):
 				if opt[0] == 'multi-set':
 					optval = opt[3]
 				elif opt[0] == 'set':
-					optval = [opt[3]]
+					optval = opt[3:]
 				else:
 					continue
-				for optval in optval:
-					try:
-						if opt[2] == "prefregex":
+				try:
+					if opt[2] == "prefregex":
+						for optval in optval:
 							self._filter.prefRegex = optval
-						elif opt[2] == "addfailregex":
-							stor = regex_values.get('fail')
-							if not stor: stor = regex_values['fail'] = list()
+					elif opt[2] == "addfailregex":
+						stor = regex_values.get('fail')
+						if not stor: stor = regex_values['fail'] = list()
+						for optval in optval:
 							stor.append(RegexStat(optval))
 							#self._filter.addFailRegex(optval)
-						elif opt[2] == "addignoreregex":
-							stor = regex_values.get('ignore')
-							if not stor: stor = regex_values['ignore'] = list()
+					elif opt[2] == "addignoreregex":
+						stor = regex_values.get('ignore')
+						if not stor: stor = regex_values['ignore'] = list()
+						for optval in optval:
 							stor.append(RegexStat(optval))
 							#self._filter.addIgnoreRegex(optval)
-						elif opt[2] == "maxlines":
+					elif opt[2] == "maxlines":
+						for optval in optval:
 							self.setMaxLines(optval)
-						elif opt[2] == "datepattern":
+					elif opt[2] == "datepattern":
+						for optval in optval:
 							self.setDatePattern(optval)
-						elif opt[2] == "addjournalmatch":
+					elif opt[2] == "addjournalmatch":
+						if self._opts.journalmatch is None:
 							self.setJournalMatch(optval)
-					except ValueError as e: # pragma: no cover
-						output( "ERROR: Invalid value for %s (%r) " \
-							  "read from %s: %s" % (opt[2], optval, value, e) )
-						return False
+				except ValueError as e: # pragma: no cover
+					output( "ERROR: Invalid value for %s (%r) " \
+						  "read from %s: %s" % (opt[2], optval, value, e) )
+					return False
 
 		else:
 			output( "Use %11s line : %s" % (regex, shortstr(value)) )
@@ -510,7 +515,7 @@ class Fail2banRegex(object):
 		for line in hdlr:
 			yield self.decode_line(line)
 
-	def start(self, opts, args):
+	def start(self, args):
 
 		cmd_log, cmd_regex = args[:2]
 
@@ -603,5 +608,5 @@ def exec_command_line(*args):
 	logSys.addHandler(stdout)
 
 	fail2banRegex = Fail2banRegex(opts)
-	if not fail2banRegex.start(opts, args):
+	if not fail2banRegex.start(args):
 		sys.exit(-1)
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 5767a3f7..8e6fcfe0 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -96,7 +96,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"test", r".** from <HOST>$"
 		)
-		self.assertFalse(fail2banRegex.start(opts, args))
+		self.assertFalse(fail2banRegex.start(args))
 		self.assertLogged("Unable to compile regular expression")
 
 	def testWrongIngnoreRE(self):
@@ -104,7 +104,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--datepattern", "{^LN-BEG}EPOCH",
 			"test", r".*? from <HOST>$", r".**"
 		)
-		self.assertFalse(fail2banRegex.start(opts, args))
+		self.assertFalse(fail2banRegex.start(args))
 		self.assertLogged("Unable to compile regular expression")
 
 	def testDirectFound(self):
@@ -114,7 +114,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
 			r"Authentication failure for .*? from <HOST>$"
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
 
 	def testDirectNotFound(self):
@@ -123,7 +123,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
 			r"XYZ from <HOST>$"
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed')
 
 	def testDirectIgnored(self):
@@ -133,7 +133,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			r"Authentication failure for .*? from <HOST>$",
 			r"kevin from 192.0.2.0$"
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed')
 
 	def testDirectRE_1(self):
@@ -143,7 +143,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed')
 
 		self.assertLogged('Error decoding line');
@@ -159,7 +159,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed')
 
 	def testDirectRE_1raw_noDns(self):
@@ -169,7 +169,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed')
 
 	def testDirectRE_2(self):
@@ -179,7 +179,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed')
 
 	def testVerbose(self):
@@ -189,7 +189,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed')
 
 		self.assertLogged('141.3.81.106  Sun Aug 14 11:53:59 2005')
@@ -200,7 +200,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
 
 		self.assertLogged('Error decoding line')
@@ -215,7 +215,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"--debuggex", "--print-all-matched",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
-		self.assertTrue(fail2banRegex.start(opts, args))
+		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
 
 		self.assertLogged('https://')

From 35efca594136695f537760e417ab4255a00692c3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 22 Feb 2017 18:39:44 +0100
Subject: [PATCH 310/623] Better multi-line handling introduced: single-line
 parsing with caching of needed failure information to process in further
 lines. Many times faster and fewer CPU-hungry because of parsing with
 `maxlines=1`, so without line buffering (scrolling of the buffer-window).
 Combination of tags `<F-MLFID>` and `<F-NOFAIL>` can be used now to process
 multi-line logs using single-line expressions: - tag `<F-MLFID>`: used to
 identify resp. store failure info for groups of log-lines with the same
 identifier (e. g. combined failure-info for the same conn-id by
 `<F-MLFID>(?:conn-id)</F-MLFID>`, see sshd.conf for example) - tag
 `<F-NOFAIL>`: used as mark for no-failure (helper to accumulate common
 failure-info); filter.d/sshd.conf: [sshd], [sshd-ddos], [sshd-aggressive]
 optimized with pre-filtering using new option `prefregex` and new multi-line
 handling.

---
 config/filter.d/sshd.conf         |  58 ++++----
 fail2ban/server/failregex.py      |   8 +-
 fail2ban/server/filter.py         | 222 ++++++++++++++++++------------
 fail2ban/server/ticket.py         |   9 +-
 fail2ban/server/utils.py          |   6 +
 fail2ban/tests/filtertestcase.py  |   1 +
 fail2ban/tests/samplestestcase.py |  52 ++++---
 7 files changed, 214 insertions(+), 142 deletions(-)

diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 872a73e4..922ea193 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -24,37 +24,37 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)?
 __suff = (?: \[preauth\])?\s*
 __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
 
-# single line prefix:
-__prefix_line_sl = %(__prefix_line)s%(__pref)s
-# multi line prefixes (for first and second lines):
-__prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s
-__prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
+prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
 
 mode = %(normal)s
 
-normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-         ^%(__prefix_line_sl)s<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
-         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
-         ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)\s*%(__suff)s$
-         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
-         ^%(__prefix_line_sl)sUser <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
-         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
-         ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
-         ^%(__prefix_line_ml1)sUser <F-USER>.+</F-USER> not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
-         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
-         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for <F-USER>.+</F-USER>%(__suff)s$
+normal = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
+         ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
+         ^Failed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+         ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
+         ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+         ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+         ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
+         ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+         ^Received disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
+         ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+         ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+         ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
+         ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+         ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
+         ^Disconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__suff)s
+         ^<F-NOFAIL>Received disconnect</F-NOFAIL> from <HOST>: 11:
+         ^<F-NOFAIL>Connection closed</F-NOFAIL> by <HOST>%(__suff)s$
 
-ddos =   ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
-         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
-         ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
-         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
-         ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
+ddos =   ^Did not receive identification string from <HOST>%(__suff)s$
+         ^Received disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+         ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
+         ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+         ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
+         ^Read from socket failed: Connection reset by peer \[preauth\]
+
+common = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
 
 aggressive = %(normal)s
              %(ddos)s
@@ -62,11 +62,11 @@ aggressive = %(normal)s
 [Definition]
 
 failregex = %(mode)s
+            %(common)s
 
 ignoreregex = 
 
-# "maxlines" is number of log lines to buffer for multi-line regex searches
-maxlines = 10
+maxlines = 1
 
 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
 
diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 19a89303..59f59978 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -323,6 +323,10 @@ class RegexException(Exception):
 #
 FAILURE_ID_GROPS = ("fid", "ip4", "ip6", "dns")
 
+# Additionally allows multi-line failure-id (used for wrapping e. g. conn-id to host)
+#
+FAILURE_ID_PRESENTS = FAILURE_ID_GROPS + ("mlfid",)
+
 ##
 # Regular expression class.
 #
@@ -341,9 +345,9 @@ class FailRegex(Regex):
 		# Initializes the parent.
 		Regex.__init__(self, regex, **kwargs)
 		# Check for group "dns", "ip4", "ip6", "fid"
-		if (not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]
+		if (not [grp for grp in FAILURE_ID_PRESENTS if grp in self._regexObj.groupindex]
 			and (prefRegex is None or
-				not [grp for grp in FAILURE_ID_GROPS if grp in prefRegex._regexObj.groupindex])
+				not [grp for grp in FAILURE_ID_PRESENTS if grp in prefRegex._regexObj.groupindex])
 		):
 			raise RegexException("No failure-id group in '%s'" % self._regex)
 	
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 85c07103..6b782fcc 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -38,6 +38,7 @@ from .datedetector import DateDetector
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
+from .utils import Utils
 from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
@@ -88,6 +89,8 @@ class Filter(JailThread):
 		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
 		self.__encoding = PREFER_ENC
+		## Cache temporary holds failures info (used by multi-line for wrapping e. g. conn-id to host):
+		self.__mlfidCache = None
 		## Error counter (protected, so can be used in filter implementations)
 		## if it reached 100 (at once), run-cycle will go idle
 		self._errors = 0
@@ -101,7 +104,7 @@ class Filter(JailThread):
 		self.ticks = 0
 
 		self.dateDetector = DateDetector()
-		logSys.debug("Created %s" % self)
+		logSys.debug("Created %s", self)
 
 	def __repr__(self):
 		return "%s(%r)" % (self.__class__.__name__, self.jail)
@@ -131,6 +134,13 @@ class Filter(JailThread):
 					self.delLogPath(path)
 				delattr(self, '_reload_logs')
 
+	@property
+	def mlfidCache(self):
+		if self.__mlfidCache:
+			return self.__mlfidCache
+		self.__mlfidCache = Utils.Cache(maxCount=100, maxTime=5*60)
+		return self.__mlfidCache
+
 	@property
 	def prefRegex(self):
 		return self.__prefRegex
@@ -170,7 +180,7 @@ class Filter(JailThread):
 			del self.__failRegex[index]
 		except IndexError:
 			logSys.error("Cannot remove regular expression. Index %d is not "
-						 "valid" % index)
+						 "valid", index)
 
 	##
 	# Get the regular expression which matches the failure.
@@ -208,7 +218,7 @@ class Filter(JailThread):
 			del self.__ignoreRegex[index]
 		except IndexError:
 			logSys.error("Cannot remove regular expression. Index %d is not "
-						 "valid" % index)
+						 "valid", index)
 
 	##
 	# Get the regular expression which matches the failure.
@@ -231,9 +241,9 @@ class Filter(JailThread):
 		value = value.lower()			  # must be a string by now
 		if not (value in ('yes', 'warn', 'no', 'raw')):
 			logSys.error("Incorrect value %r specified for usedns. "
-						 "Using safe 'no'" % (value,))
+						 "Using safe 'no'", value)
 			value = 'no'
-		logSys.debug("Setting usedns = %s for %s" % (value, self))
+		logSys.debug("Setting usedns = %s for %s", value, self)
 		self.__useDns = value
 
 	##
@@ -346,7 +356,7 @@ class Filter(JailThread):
 			encoding = PREFER_ENC
 		codecs.lookup(encoding) # Raise LookupError if invalid codec
 		self.__encoding = encoding
-		logSys.info("  encoding: %s" % encoding)
+		logSys.info("  encoding: %s", encoding)
 		return encoding
 
 	##
@@ -391,7 +401,7 @@ class Filter(JailThread):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
 		if self.inIgnoreIPList(ip):
-			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
+			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
 
 		unixTime = MyTime.time()
 		self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry())
@@ -435,7 +445,7 @@ class Filter(JailThread):
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
 		if log_ignore:
-			logSys.info("[%s] Ignore %s by %s" % (self.jailName, ip, ignore_source))
+			logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source)
 
 	def getIgnoreIP(self):
 		return self.__ignoreIpList
@@ -459,7 +469,7 @@ class Filter(JailThread):
 
 		if self.__ignoreCommand:
 			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
-			logSys.debug('ignore command: ' + command)
+			logSys.debug('ignore command: %s', command)
 			ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1))
 			ret_ignore = ret and ret_ignore == 0
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
@@ -498,10 +508,7 @@ class Filter(JailThread):
 			for element in self.processLine(line, date):
 				ip = element[1]
 				unixTime = element[2]
-				lines = element[3]
-				fail = {}
-				if len(element) > 4:
-					fail = element[4]
+				fail = element[3]
 				logSys.debug("Processing line with time:%s and ip:%s", 
 						unixTime, ip)
 				if self.inIgnoreIPList(ip, log_ignore=True):
@@ -509,7 +516,7 @@ class Filter(JailThread):
 				logSys.info(
 					"[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 				)
-				tick = FailTicket(ip, unixTime, lines, data=fail)
+				tick = FailTicket(ip, unixTime, data=fail)
 				self.failManager.addFailure(tick)
 			# reset (halve) error counter (successfully processed line):
 			if self._errors:
@@ -544,6 +551,29 @@ class Filter(JailThread):
 				return ignoreRegexIndex
 		return None
 
+	def _mergeFailure(self, mlfid, fail, failRegex):
+		mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
+		if mlfidFail:
+			mlfidGroups = mlfidFail[1]
+			# if current line not failure, but previous was failure:
+			if fail.get('nofail') and not mlfidGroups.get('nofail'):
+				del fail['nofail'] # remove nofail flag - was already market as failure
+				self.mlfidCache.unset(mlfid) # remove cache entry
+			# if current line is failure, but previous was not:
+			elif not fail.get('nofail') and mlfidGroups.get('nofail'):
+				del mlfidGroups['nofail'] # remove nofail flag
+				self.mlfidCache.unset(mlfid) # remove cache entry
+			fail2 = mlfidGroups.copy()
+			fail2.update(fail)
+			fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
+			fail = fail2
+		elif fail.get('nofail'):
+			fail["matches"] = failRegex.getMatchedTupleLines()
+			mlfidFail = [self.__lastDate, fail]
+			self.mlfidCache.set(mlfid, mlfidFail)
+		return fail
+
+
 	##
 	# Finds the failure in a line given split into time and log parts.
 	#
@@ -618,76 +648,94 @@ class Filter(JailThread):
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
 			failRegex.search(self.__lineBuffer, orgBuffer)
-			if failRegex.hasMatched():
-				# The failregex matched.
-				logSys.log(7, "Matched %s", failRegex)
-				# Checks if we must ignore this match.
-				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
-						is not None:
-					# The ignoreregex matched. Remove ignored match.
-					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-					logSys.log(7, "Matched ignoreregex and was ignored")
-					if not self.checkAllRegex:
-						break
-					else:
-						continue
-				if date is None:
-					logSys.warning(
-						"Found a match for %r but no valid date/time "
-						"found for %r. Please try setting a custom "
-						"date pattern (see man page jail.conf(5)). "
-						"If format is complex, please "
-						"file a detailed issue on"
-						" https://github.com/fail2ban/fail2ban/issues "
-						"in order to get support for this format."
-						 % ("\n".join(failRegex.getMatchedLines()), timeText))
+			if not failRegex.hasMatched():
+				continue
+			# The failregex matched.
+			logSys.log(7, "Matched %s", failRegex)
+			# Checks if we must ignore this match.
+			if self.ignoreLine(failRegex.getMatchedTupleLines()) \
+					is not None:
+				# The ignoreregex matched. Remove ignored match.
+				self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+				logSys.log(7, "Matched ignoreregex and was ignored")
+				if not self.checkAllRegex:
+					break
 				else:
-					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-					# retrieve failure-id, host, etc from failure match:
-					raw = returnRawHost
-					try:
-						if preGroups:
-							fail = preGroups.copy()
-							fail.update(failRegex.getGroups())
-						else:
-							fail = failRegex.getGroups()
-						# failure-id:
-						fid = fail.get('fid')
-						# ip-address or host:
-						host = fail.get('ip4')
-						if host is not None:
-							cidr = IPAddr.FAM_IPv4
-							raw = True
-						else:
-							host = fail.get('ip6')
-							if host is not None:
-								cidr = IPAddr.FAM_IPv6
-								raw = True
-						if host is None:
-							host = fail.get('dns')
-							if host is None:
+					continue
+			if date is None:
+				logSys.warning(
+					"Found a match for %r but no valid date/time "
+					"found for %r. Please try setting a custom "
+					"date pattern (see man page jail.conf(5)). "
+					"If format is complex, please "
+					"file a detailed issue on"
+					" https://github.com/fail2ban/fail2ban/issues "
+					"in order to get support for this format.",
+					 "\n".join(failRegex.getMatchedLines()), timeText)
+				continue
+			self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+			# retrieve failure-id, host, etc from failure match:
+			try:
+				raw = returnRawHost
+				if preGroups:
+					fail = preGroups.copy()
+					fail.update(failRegex.getGroups())
+				else:
+					fail = failRegex.getGroups()
+				# first try to check we have mlfid case (caching of connection id by multi-line):
+				mlfid = fail.get('mlfid')
+				if mlfid is not None:
+					fail = self._mergeFailure(mlfid, fail, failRegex)
+				else:
+					# matched lines:
+					fail["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
+				# failure-id:
+				fid = fail.get('fid')
+				# ip-address or host:
+				host = fail.get('ip4')
+				if host is not None:
+					cidr = IPAddr.FAM_IPv4
+					raw = True
+				else:
+					host = fail.get('ip6')
+					if host is not None:
+						cidr = IPAddr.FAM_IPv6
+						raw = True
+				if host is None:
+					host = fail.get('dns')
+					if host is None:
+						# first try to check we have mlfid case (cache connection id):
+						if fid is None:
+							if mlfid:
+								fail = self._mergeFailure(mlfid, fail, failRegex)
+							else:
 								# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
-								if fid is None:
-									fid = failRegex.getFailID()
-								host = fid
-								cidr = IPAddr.CIDR_RAW
-						# if raw - add single ip or failure-id,
-						# otherwise expand host to multiple ips using dns (or ignore it if not valid):
-						if raw:
-							ip = IPAddr(host, cidr)
-							# check host equal failure-id, if not - failure with complex id:
-							if fid is not None and fid != host:
-								ip = IPAddr(fid, IPAddr.CIDR_RAW)
-							ips = [ip]
-						else:
-							ips = DNSUtils.textToIp(host, self.__useDns)
-						for ip in ips:
-							failList.append([failRegexIndex, ip, date,
-								failRegex.getMatchedLines(), fail])
-						if not self.checkAllRegex:
-							break
-					except RegexException as e: # pragma: no cover - unsure if reachable
-						logSys.error(e)
+								fid = failRegex.getFailID()
+						host = fid
+						cidr = IPAddr.CIDR_RAW
+				# if mlfid case (not failure):
+				if host is None:
+					if not self.checkAllRegex: # or fail.get('nofail'):
+					 	return failList
+					ips = [None]
+				# if raw - add single ip or failure-id,
+				# otherwise expand host to multiple ips using dns (or ignore it if not valid):
+				elif raw:
+					ip = IPAddr(host, cidr)
+					# check host equal failure-id, if not - failure with complex id:
+					if fid is not None and fid != host:
+						ip = IPAddr(fid, IPAddr.CIDR_RAW)
+					ips = [ip]
+				# otherwise, try to use dns conversion:
+				else:
+					ips = DNSUtils.textToIp(host, self.__useDns)
+				# append failure with match to the list:
+				for ip in ips:
+					failList.append([failRegexIndex, ip, date, fail])
+				if not self.checkAllRegex:
+					break
+			except RegexException as e: # pragma: no cover - unsure if reachable
+				logSys.error(e)
 		return failList
 
 	def status(self, flavor="basic"):
@@ -751,7 +799,7 @@ class FileFilter(Filter):
 		db = self.jail.database
 		if db is not None:
 			db.updateLog(self.jail, log)
-		logSys.info("Removed logfile: %r" % path)
+		logSys.info("Removed logfile: %r", path)
 		self._delLogPath(path)
 		return
 
@@ -816,7 +864,7 @@ class FileFilter(Filter):
 	def getFailures(self, filename):
 		log = self.getLog(filename)
 		if log is None:
-			logSys.error("Unable to get failures in " + filename)
+			logSys.error("Unable to get failures in %s", filename)
 			return False
 		# We should always close log (file), otherwise may be locked (log-rotate, etc.)
 		try:
@@ -825,11 +873,11 @@ class FileFilter(Filter):
 				has_content = log.open()
 			# see http://python.org/dev/peps/pep-3151/
 			except IOError as e:
-				logSys.error("Unable to open %s" % filename)
+				logSys.error("Unable to open %s", filename)
 				logSys.exception(e)
 				return False
 			except OSError as e: # pragma: no cover - requires race condition to tigger this
-				logSys.error("Error opening %s" % filename)
+				logSys.error("Error opening %s", filename)
 				logSys.exception(e)
 				return False
 			except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
@@ -1050,7 +1098,7 @@ class FileContainer:
 		## sys.stdout.flush()
 		# Compare hash and inode
 		if self.__hash != myHash or self.__ino != stats.st_ino:
-			logSys.info("Log rotation detected for %s" % self.__filename)
+			logSys.info("Log rotation detected for %s", self.__filename)
 			self.__hash = myHash
 			self.__ino = stats.st_ino
 			self.__pos = 0
diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py
index a66b64ac..be205303 100644
--- a/fail2ban/server/ticket.py
+++ b/fail2ban/server/ticket.py
@@ -138,7 +138,8 @@ class Ticket(object):
 		self._data['matches'] = matches or []
 
 	def getMatches(self):
-		return self._data.get('matches', [])
+		return [(line if isinstance(line, basestring) else "".join(line)) \
+			for line in self._data.get('matches', ())]
 
 	@property
 	def restored(self):
@@ -235,7 +236,11 @@ class FailTicket(Ticket):
 		self.__retry += count
 		self._data['failures'] += attempt
 		if matches:
-			self._data['matches'] += matches
+			# we should duplicate "matches", because possibly referenced to multiple tickets:
+			if self._data['matches']:
+				self._data['matches'] = self._data['matches'] + matches
+			else:
+				self._data['matches'] = matches
 
 	def setLastTime(self, value):
 		if value > self._time:
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 57da495a..b258ae77 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -98,6 +98,12 @@ class Utils():
 					cache.popitem()
 			cache[k] = (v, t + self.maxTime)
 
+		def unset(self, k):
+			try:
+				del self._cache[k]
+			except KeyError: # pragme: no cover
+				pass
+
 
 	@staticmethod
 	def setFBlockMode(fhandle, value):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 174152b9..8558f8d3 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1431,6 +1431,7 @@ class GetFailures(LogCaptureTestCase):
 			('no',   output_no),
 			('warn', output_yes)
 		):
+			self.pruneLog("[test-phase useDns=%s]" % useDns)
 			jail = DummyJail()
 			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.active = True
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index a97e92c3..b59ccd98 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -150,29 +150,34 @@ def testSampleRegexsFactory(name, basedir):
 				else:
 					faildata = {}
 
-				ret = self.filter.processLine(line)
-				if not ret:
-					# Check line is flagged as none match
-					self.assertFalse(faildata.get('match', True),
-						 "Line not matched when should have: %s:%i, line:\n%s" %
-						(logFile.filename(), logFile.filelineno(), line))
-				elif ret:
-					# Check line is flagged to match
-					self.assertTrue(faildata.get('match', False),
-						"Line matched when shouldn't have: %s:%i, line:\n%s" %
-						(logFile.filename(), logFile.filelineno(), line))
-					self.assertEqual(len(ret), 1, "Multiple regexs matched %r - %s:%i" %
-									 (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
+				try:
+					ret = self.filter.processLine(line)
+					if not ret:
+						# Check line is flagged as none match
+						self.assertFalse(faildata.get('match', True),
+							"Line not matched when should have")
+						continue
 
-					# Verify timestamp and host as expected
-					failregex, host, fail2banTime, lines, fail = ret[0]
-					self.assertEqual(host, faildata.get("host", None))
-					# Verify other captures:
+					failregex, fid, fail2banTime, fail = ret[0]
+					# Bypass no failure helpers-regexp:
+					if not faildata.get('match', False) and (fid is None or fail.get('nofail')):
+						regexsUsed.add(failregex)
+						continue
+
+					# Check line is flagged to match
+					self.assertTrue(faildata.get('match', False), 
+						"Line matched when shouldn't have")
+					self.assertEqual(len(ret), 1,
+						"Multiple regexs matched %r" % (map(lambda x: x[0], ret)))
+
+					# Fallback for backwards compatibility (previously no fid, was host only):
+					if faildata.get("host", None) is not None and fail.get("host", None) is None:
+						fail["host"] = fid
+					# Verify match captures (at least fid/host) and timestamp as expected
 					for k, v in faildata.iteritems():
-						if k not in ("time", "match", "host", "desc"):
+						if k not in ("time", "match", "desc"):
 							fv = fail.get(k, None)
-							self.assertEqual(fv, v, "Value of %s mismatch %r != %r on: %s:%i, line:\n%s" % (
-								k, fv, v, logFile.filename(), logFile.filelineno(), line))
+							self.assertEqual(fv, v)
 
 					t = faildata.get("time", None)
 					try:
@@ -185,12 +190,15 @@ def testSampleRegexsFactory(name, basedir):
 					jsonTime += jsonTimeLocal.microsecond / 1000000
 
 					self.assertEqual(fail2banTime, jsonTime,
-						"UTC Time  mismatch %s (%s) != %s (%s)  (diff %.3f seconds) on: %s:%i, line:\n%s" % 
+						"UTC Time  mismatch %s (%s) != %s (%s)  (diff %.3f seconds)" % 
 						(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
 						jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
-						fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) )
+						fail2banTime - jsonTime) )
 
 					regexsUsed.add(failregex)
+				except AssertionError as e: # pragma: no cover
+					raise AssertionError("%s on: %s:%i, line:\n%s" % (
+								e, logFile.filename(), logFile.filelineno(), line))
 
 		for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()):
 			self.assertTrue(

From 4efcc293846d18bf9bfcba90b8311607f9126d0a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 22 Feb 2017 20:45:06 +0100
Subject: [PATCH 311/623] coverage of new multi-line handling within
 fail2ban-regex

---
 fail2ban/tests/fail2banregextestcase.py | 31 ++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 8e6fcfe0..7b08b936 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -27,14 +27,14 @@ import os
 import sys
 
 from ..client import fail2banregex
-from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output
+from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output, str2LogLevel
 from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase, logSys
 from .utils import CONFIG_DIR
 
 
 fail2banregex.logSys = logSys
 def _test_output(*args):
-	logSys.info(args[0])
+	logSys.notice(args[0])
 
 fail2banregex.output = _test_output
 
@@ -45,6 +45,9 @@ DEV_NULL = None
 def _Fail2banRegex(*args):
 	parser = get_opt_parser()
 	(opts, args) = parser.parse_args(list(args))
+	# put down log-level if expected, because of too many debug-messages:
+	if opts.log_level in ("notice", "warning"):
+		logSys.setLevel(str2LogLevel(opts.log_level))
 	return (opts, args, Fail2banRegex(opts))
 
 class ExitException(Exception):
@@ -80,6 +83,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 	FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log")
 	FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log")
 
+	FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd")
 	FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf')
 
 	def setUp(self):
@@ -195,8 +199,26 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('141.3.81.106  Sun Aug 14 11:53:59 2005')
 		self.assertLogged('141.3.81.106  Sun Aug 14 11:54:59 2005')
 
-	def testWronChar(self):
+	def testVerboseFullSshd(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
+			"-v", "--verbose-date", "--print-all-matched",
+			Fail2banRegexTest.FILENAME_SSHD, Fail2banRegexTest.FILTER_SSHD
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		# test failure line and not-failure lines both presents:
+		self.assertLogged("[29116]: User root not allowed because account is locked",
+			"[29116]: Received disconnect from 1.2.3.4", all=True)
+
+	def _reset(self):
+		# reset global warn-counter:
+		from ..server.filter import _decode_line_warn
+		_decode_line_warn.clear()
+
+	def testWronChar(self):
+		self._reset()
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
 			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
@@ -210,12 +232,15 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('Nov  8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco')
 
 	def testWronCharDebuggex(self):
+		self._reset()
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
 			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--debuggex", "--print-all-matched",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
 		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Error decoding line')
 		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
 
 		self.assertLogged('https://')

From d2a3d093c6208a9d71dae27beb5e052060be481b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Feb 2017 11:54:24 +0100
Subject: [PATCH 312/623] rewritten CallingMap: performance optimized,
 immutable, self-referencing, template possibility (used in new ActionInfo
 objects); new ActionInfo handling: saves content between actions, without
 interim copying (save original on demand, recoverable via reset); test cases
 extended

---
 config/action.d/smtp.py          |   2 +-
 fail2ban/server/action.py        |  66 +++++++++++---
 fail2ban/server/actions.py       | 148 ++++++++++++++++++-------------
 fail2ban/tests/actiontestcase.py |  48 +++++++++-
 fail2ban/tests/servertestcase.py |   2 +-
 5 files changed, 186 insertions(+), 80 deletions(-)

diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py
index aa85b259..9cdfe327 100644
--- a/config/action.d/smtp.py
+++ b/config/action.d/smtp.py
@@ -123,7 +123,7 @@ class SMTPAction(ActionBase):
 		self.message_values = CallingMap(
 			jailname = self._jail.name,
 			hostname = socket.gethostname,
-			bantime = self._jail.actions.getBanTime,
+			bantime = lambda: self._jail.actions.getBanTime(),
 			)
 
 		# bypass ban/unban for restored tickets
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index bf291690..e7a98dbb 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -50,9 +50,14 @@ allowed_ipv6 = True
 # capture groups from filter for map to ticket data:
 FCUSTAG_CRE = re.compile(r'<F-([A-Z0-9_\-]+)>'); # currently uppercase only
 
+# New line, space
+ADD_REPL_TAGS = {
+  "br": "\n", 
+  "sp": " "
+}
 
 
-class CallingMap(MutableMapping):
+class CallingMap(MutableMapping, object):
 	"""A Mapping type which returns the result of callable values.
 
 	`CallingMap` behaves similar to a standard python dictionary,
@@ -69,23 +74,64 @@ class CallingMap(MutableMapping):
 		The dictionary data which can be accessed to obtain items uncalled
 	"""
 
+	# immutable=True saves content between actions, without interim copying (save original on demand, recoverable via reset)
+	__slots__ = ('data', 'storage', 'immutable', '__org_data')
 	def __init__(self, *args, **kwargs):
+		self.storage = dict()
+		self.immutable = True
 		self.data = dict(*args, **kwargs)
 
+	def reset(self, immutable=True):
+		self.storage = dict()
+		try:
+			self.data = self.__org_data
+		except AttributeError:
+			pass
+		self.immutable = immutable
+
 	def __repr__(self):
-		return "%s(%r)" % (self.__class__.__name__, self.data)
+		return "%s(%r)" % (self.__class__.__name__, self._asdict())
+
+	def _asdict(self):
+		try:
+			return dict(self)
+		except:
+			return dict(self.data, **self.storage)
 
 	def __getitem__(self, key):
-		value = self.data[key]
+		try:
+			value = self.storage[key]
+		except KeyError:
+			value = self.data[key]
 		if callable(value):
-			value = value()
-			self.data[key] = value
+			# check arguments can be supplied to callable (for backwards compatibility):
+			value = value(self) if hasattr(value, '__code__') and value.__code__.co_argcount else value()
+			self.storage[key] = value
 		return value
 
 	def __setitem__(self, key, value):
-		self.data[key] = value
+		# mutate to copy:
+		if self.immutable:
+			self.storage = self.storage.copy()
+			self.__org_data = self.data
+			self.data = self.data.copy()
+			self.immutable = False
+		self.storage[key] = value
+
+	def __unavailable(self, key):
+		raise KeyError("Key %r was deleted" % key)
 
 	def __delitem__(self, key):
+		# mutate to copy:
+		if self.immutable:
+			self.storage = self.storage.copy()
+			self.__org_data = self.data
+			self.data = self.data.copy()
+			self.immutable = False
+		try:
+			del self.storage[key]
+		except KeyError:
+			pass
 		del self.data[key]
 
 	def __iter__(self):
@@ -94,7 +140,7 @@ class CallingMap(MutableMapping):
 	def __len__(self):
 		return len(self.data)
 
-	def copy(self):
+	def copy(self): # pargma: no cover
 		return self.__class__(self.data.copy())
 
 
@@ -436,9 +482,6 @@ class CommandAction(ActionBase):
 		# interpolation of dictionary:
 		if subInfo is None:
 			subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
-		# New line, space
-		for (tag, value) in (("br", '\n'), ("sp", " ")):
-			if subInfo.get(tag) is None: subInfo[tag] = value
 		# cache if possible:
 		if csubkey is not None:
 			cache[csubkey] = subInfo
@@ -453,7 +496,8 @@ class CommandAction(ActionBase):
 			if value is None:
 				value = subInfo.get(tag)
 				if value is None:
-					return m.group()	# fallback (no replacement)
+					# fallback (no or default replacement)
+					return ADD_REPL_TAGS.get(tag, m.group())
 			value = str(value)		# assure string
 			if tag in cls._escapedTags:
 				# That one needs to be escaped since its content is
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 3a5eada9..6b793b8f 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -286,44 +286,86 @@ class Actions(JailThread, Mapping):
 		self.stopActions()
 		return True
 
-	def __getBansMerged(self, mi, overalljails=False):
-		"""Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside.
+	class ActionInfo(CallingMap):
 
-		This function never returns None for ainfo lambdas - always a ticket (merged or single one)
-		and prevents any errors through merging (to guarantee ban actions will be executed).
-		[TODO] move merging to observer - here we could wait for merge and read already merged info from a database
+		AI_DICT = {
+			"ip":				lambda self: self.__ticket.getIP(),
+			"ip-rev":		lambda self: self['ip'].getPTR(''),
+			"fid":			lambda self: self.__ticket.getID(),
+			"failures":	lambda self: self.__ticket.getAttempt(),
+			"time":			lambda self: self.__ticket.getTime(),
+			"matches":	lambda self: "\n".join(self.__ticket.getMatches()),
+			# to bypass actions, that should not be executed for restored tickets
+			"restored":	lambda self: (1 if self.__ticket.restored else 0),
+			# extra-interpolation - all match-tags (captured from the filter):
+			"F-*":			lambda self, tag=None: self.__ticket.getData(tag),
+			# merged info:
+			"ipmatches":			lambda self: "\n".join(self._mi4ip(True).getMatches()),
+			"ipjailmatches":	lambda self: "\n".join(self._mi4ip().getMatches()),
+			"ipfailures":			lambda self: self._mi4ip(True).getAttempt(),
+			"ipjailfailures":	lambda self: self._mi4ip().getAttempt(),
+		}
 
-		Parameters
-		----------
-		mi : dict
-			merge info, initial for lambda should contains {ip, ticket}
-		overalljails : bool
-			switch to get a merged bans :
-			False - (default) bans merged for current jail only
-			True - bans merged for all jails of current ip address
+		__slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip')
+
+		def __init__(self, ticket, jail=None, immutable=True, data=AI_DICT):
+			self.__ticket = ticket
+			self.__jail = jail
+			self.storage = dict()
+			self.immutable = immutable
+			self.data = data
+		
+		def copy(self): # pargma: no cover
+			return self.__class__(self.__ticket, self.__jail, self.immutable, self.data.copy())
+
+		def _mi4ip(self, overalljails=False):
+			"""Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside.
+
+			This function never returns None for ainfo lambdas - always a ticket (merged or single one)
+			and prevents any errors through merging (to guarantee ban actions will be executed).
+			[TODO] move merging to observer - here we could wait for merge and read already merged info from a database
+
+			Parameters
+			----------
+			overalljails : bool
+				switch to get a merged bans :
+				False - (default) bans merged for current jail only
+				True - bans merged for all jails of current ip address
+
+			Returns
+			-------
+			BanTicket 
+				merged or self ticket only
+			"""
+			if not hasattr(self, '__mi4ip'):
+				self.__mi4ip = {}
+			mi = self.__mi4ip
+			idx = 'all' if overalljails else 'jail'
+			if idx in mi:
+				return mi[idx] if mi[idx] is not None else self.__ticket
+			try:
+				jail = self.__jail
+				ip = self['ip']
+				mi[idx] = None
+				if not jail.database: # pragma: no cover
+					return self.__ticket
+				if overalljails:
+					mi[idx] = jail.database.getBansMerged(ip=ip)
+				else:
+					mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail)
+			except Exception as e:
+				logSys.error(
+					"Failed to get %s bans merged, jail '%s': %s",
+					idx, jail.name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			return mi[idx] if mi[idx] is not None else self.__ticket
+
+
+	def __getActionInfo(self, ticket):
+		ip = ticket.getIP()
+		aInfo = Actions.ActionInfo(ticket, self._jail)
+		return aInfo
 
-		Returns
-		-------
-		BanTicket 
-			merged or self ticket only
-		"""
-		idx = 'all' if overalljails else 'jail'
-		if idx in mi:
-			return mi[idx] if mi[idx] is not None else mi['ticket']
-		try:
-			jail=self._jail
-			ip=mi['ip']
-			mi[idx] = None
-			if overalljails:
-				mi[idx] = jail.database.getBansMerged(ip=ip)
-			else:
-				mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail)
-		except Exception as e:
-			logSys.error(
-				"Failed to get %s bans merged, jail '%s': %s",
-				idx, jail.name, e,
-				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-		return mi[idx] if mi[idx] is not None else mi['ticket']
 
 	def __checkBan(self):
 		"""Check for IP address to ban.
@@ -343,24 +385,7 @@ class Actions(JailThread, Mapping):
 				break
 			bTicket = BanManager.createBanTicket(ticket)
 			ip = bTicket.getIP()
-			aInfo = CallingMap({
-				"ip"			:	ip,
-				"ip-rev"	:	lambda: ip.getPTR(''),
-				"failures":	bTicket.getAttempt(),
-				"time"		:	bTicket.getTime(),
-				"matches"	:	"\n".join(bTicket.getMatches()),
-				# to bypass actions, that should not be executed for restored tickets
-				"restored":	(1 if ticket.restored else 0),
-				# extra-interpolation - all match-tags (captured from the filter):
-				"F-*": lambda tag=None: bTicket.getData(tag)
-			})
-			if self._jail.database is not None:
-				mi4ip = lambda overalljails=False, self=self, \
-					mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
-				aInfo["ipmatches"]      = lambda: "\n".join(mi4ip(True).getMatches())
-				aInfo["ipjailmatches"]  = lambda: "\n".join(mi4ip().getMatches())
-				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
-				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
+			aInfo = self.__getActionInfo(bTicket)
 			reason = {}
 			if self.__banManager.addBanTicket(bTicket, reason=reason):
 				cnt += 1
@@ -369,7 +394,8 @@ class Actions(JailThread, Mapping):
 					try:
 						if ticket.restored and getattr(action, 'norestored', False):
 							continue
-						action.ban(aInfo.copy())
+						if not aInfo.immutable: aInfo.reset()
+						action.ban(aInfo)
 					except Exception as e:
 						logSys.error(
 							"Failed to execute ban jail '%s' action '%s' "
@@ -452,21 +478,17 @@ class Actions(JailThread, Mapping):
 			unbactions = self._actions
 		else:
 			unbactions = actions
-		aInfo = dict()
-		aInfo["ip"] = ticket.getIP()
-		aInfo["failures"] = ticket.getAttempt()
-		aInfo["time"] = ticket.getTime()
-		aInfo["matches"] = "".join(ticket.getMatches())
-		# to bypass actions, that should not be executed for restored tickets
-		aInfo["restored"] = 1 if ticket.restored else 0
+		ip = ticket.getIP()
+		aInfo = self.__getActionInfo(ticket)
 		if actions is None:
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
 			try:
 				if ticket.restored and getattr(action, 'norestored', False):
 					continue
-				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"])
-				action.unban(aInfo.copy())
+				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip)
+				if not aInfo.immutable: aInfo.reset()
+				action.unban(aInfo)
 			except Exception as e:
 				logSys.error(
 					"Failed to execute unban jail '%s' action '%s' "
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index ca908b65..8834ab8d 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -194,7 +194,7 @@ class CommandActionTest(LogCaptureTestCase):
 		# Callable
 		self.assertEqual(
 			self.__action.replaceTag("09 <matches> 11",
-				CallingMap(matches=lambda: str(10))),
+				CallingMap(matches=lambda self: str(10))),
 			"09 10 11")
 
 	def testReplaceNoTag(self):
@@ -202,7 +202,7 @@ class CommandActionTest(LogCaptureTestCase):
 		# Will raise ValueError if it is
 		self.assertEqual(
 			self.__action.replaceTag("abc",
-				CallingMap(matches=lambda: int("a"))), "abc")
+				CallingMap(matches=lambda self: int("a"))), "abc")
 
 	def testReplaceTagSelfRecursion(self):
 		setattr(self.__action, 'a', "<a")
@@ -332,7 +332,7 @@ class CommandActionTest(LogCaptureTestCase):
 		aInfo = CallingMap({
 			'ABC': "123",
 			'ip': '192.0.2.1',
-			'F-*': lambda: {
+			'F-*': lambda self: {
 		  	'fid': 111,
 		  	'fport': 222,
 				'user': "tester"
@@ -442,7 +442,7 @@ class CommandActionTest(LogCaptureTestCase):
 			"stderr: 'The rain in Spain stays mainly in the plain'\n")
 
 	def testCallingMap(self):
-		mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'),
+		mymap = CallingMap(callme=lambda self: str(10), error=lambda self: int('a'),
 			dontcallme= "string", number=17)
 
 		# Should work fine
@@ -451,3 +451,43 @@ class CommandActionTest(LogCaptureTestCase):
 			"10 okay string 17")
 		# Error will now trip, demonstrating delayed call
 		self.assertRaises(ValueError, lambda x: "%(error)i" % x, mymap)
+
+	def testCallingMapModify(self):
+		m = CallingMap({
+			'a': lambda self: 2 + 3,
+			'b': lambda self: self['a'] + 6,
+			'c': 'test',
+		})
+		# test reset (without modifications):
+		m.reset()
+		# do modifications:
+		m['a'] = 4
+		del m['c']
+		# test set and delete:
+		self.assertEqual(len(m), 2)
+		self.assertNotIn('c', m)
+		self.assertEqual((m['a'], m['b']), (4, 10))
+		# reset to original and test again:
+		m.reset()
+		s = repr(m)
+		self.assertEqual(len(m), 3)
+		self.assertIn('c', m)
+		self.assertEqual((m['a'], m['b'], m['c']), (5, 11, 'test'))
+
+	def testCallingMapRep(self):
+		m = CallingMap({
+			'a': lambda self: 2 + 3,
+			'b': lambda self: self['a'] + 6,
+			'c': ''
+		})
+		s = repr(m)
+		self.assertIn("'a': 5", s)
+		self.assertIn("'b': 11", s)
+		self.assertIn("'c': ''", s)
+		
+		m['c'] = lambda self: self['xxx'] + 7; # unresolvable
+		s = repr(m)
+		self.assertIn("'a': 5", s)
+		self.assertIn("'b': 11", s)
+		self.assertIn("'c': ", s) # presents as callable
+		self.assertNotIn("'c': ''", s) # but not empty
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index fce03573..464295fd 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1737,6 +1737,6 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						if not tests.get(test): continue
 						self.pruneLog('# === %s ===' % test)
 						ticket = _actions.CallingMap({
-							'ip': ip, 'ip-rev': lambda: ip.getPTR(''), 'failures': 100,})
+							'ip': ip, 'ip-rev': lambda self: self['ip'].getPTR(''), 'failures': 100,})
 						action.ban(ticket)
 						self.assertLogged(*tests[test], all=True)

From e4a265c75f59b45fdfa3c47e7e7ab21ef366cd3d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Feb 2017 13:22:15 +0100
Subject: [PATCH 313/623] test coverage

---
 fail2ban/client/fail2banregex.py              | 22 +++---
 .../filter.d/zzz-sshd-obsolete-multiline.conf | 76 +++++++++++++++++++
 fail2ban/tests/fail2banregextestcase.py       | 39 ++++++++++
 .../files/logs/zzz-sshd-obsolete-multiline    |  2 +
 .../files/zzz-sshd-obsolete-multiline.log     |  4 +
 5 files changed, 132 insertions(+), 11 deletions(-)
 create mode 100644 fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
 create mode 100644 fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline
 create mode 100644 fail2ban/tests/files/zzz-sshd-obsolete-multiline.log

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 395fe8ef..e2f1f67c 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -61,7 +61,7 @@ def debuggexURL(sample, regex):
 							'flavor': 'python' })
 	return 'https://www.debuggex.com/?' + q
 
-def output(args):
+def output(args): # pragma: no cover (overriden in test-cases)
 	print(args)
 
 def shortstr(s, l=53):
@@ -278,15 +278,15 @@ class Fail2banRegex(object):
 				value = os.path.splitext(os.path.basename(value))[0]
 				output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) )
 				reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir)
-				if not reader.read():
+				if not reader.read(): # pragma: no cover
 					output( "ERROR: failed to load filter %s" % value )
 					return False
-			else:
+			else: # pragma: no cover
 				## foreign file - readexplicit this file and includes if possible:
 				output( "Use %11s file : %s" % (regex, value) )
 				reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config)
 				reader.setBaseDir(None)
-				if not reader.readexplicit():
+				if not reader.readexplicit(): 
 					output( "ERROR: failed to read %s" % value )
 					return False
 			reader.getOptions(None)
@@ -298,7 +298,7 @@ class Fail2banRegex(object):
 					optval = opt[3]
 				elif opt[0] == 'set':
 					optval = opt[3:]
-				else:
+				else: # pragma: no cover
 					continue
 				try:
 					if opt[2] == "prefregex":
@@ -322,7 +322,7 @@ class Fail2banRegex(object):
 					elif opt[2] == "datepattern":
 						for optval in optval:
 							self.setDatePattern(optval)
-					elif opt[2] == "addjournalmatch":
+					elif opt[2] == "addjournalmatch": # pragma: no cover
 						if self._opts.journalmatch is None:
 							self.setJournalMatch(optval)
 				except ValueError as e: # pragma: no cover
@@ -350,7 +350,7 @@ class Fail2banRegex(object):
 			if ret is not None:
 				found = True
 				regex = self._ignoreregex[ret].inc()
-		except RegexException as e:
+		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
 			return False
 		return found
@@ -368,7 +368,7 @@ class Fail2banRegex(object):
 				regex = self._failregex[match[0]]
 				regex.inc()
 				regex.appendIP(match)
-		except RegexException as e:
+		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
 			return False
 		for bufLine in orgLineBuffer[int(fullBuffer):]:
@@ -520,9 +520,9 @@ class Fail2banRegex(object):
 		cmd_log, cmd_regex = args[:2]
 
 		try:
-			if not self.readRegex(cmd_regex, 'fail'):
+			if not self.readRegex(cmd_regex, 'fail'): # pragma: no cover
 				return False
-			if len(args) == 3 and not self.readRegex(args[2], 'ignore'):
+			if len(args) == 3 and not self.readRegex(args[2], 'ignore'): # pragma: no cover
 				return False
 		except RegexException as e:
 			output( 'ERROR: %s' % e )
@@ -534,7 +534,7 @@ class Fail2banRegex(object):
 				output( "Use         log file : %s" % cmd_log )
 				output( "Use         encoding : %s" % self._encoding )
 				test_lines = self.file_lines_gen(hdlr)
-			except IOError as e:
+			except IOError as e: # pragma: no cover
 				output( e )
 				return False
 		elif cmd_log.startswith("systemd-journal"): # pragma: no cover
diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
new file mode 100644
index 00000000..729fecd7
--- /dev/null
+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
@@ -0,0 +1,76 @@
+# Fail2Ban obsolete multiline example resp. test filter (previously sshd.conf)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = ../../../../config/filter.d/common.conf
+
+[DEFAULT]
+
+_daemon = sshd
+
+# optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
+__pref = (?:(?:error|fatal): (?:PAM: )?)?
+# optional suffix (logged from several ssh versions) like " [preauth]"
+__suff = (?: \[preauth\])?\s*
+__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
+
+# single line prefix:
+__prefix_line_sl = %(__prefix_line)s%(__pref)s
+# multi line prefixes (for first and second lines):
+__prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s
+__prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
+
+mode = %(normal)s
+
+normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__on_port_opt)s\s*$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not in any group\s*%(__suff)s$
+         ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$
+         ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+         ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
+         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$
+
+ddos =   ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
+         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+         ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+         ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
+
+aggressive = %(normal)s
+             %(ddos)s
+
+[Definition]
+
+failregex = %(mode)s
+
+ignoreregex = 
+
+# "maxlines" is number of log lines to buffer for multi-line regex searches
+maxlines = 10
+
+journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
+
+datepattern = {^LN-BEG}
+
+# DEV Notes:
+#
+#   "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because
+#   it is coming before use of <HOST> which is not hard-anchored at the end as well,
+#   and later catch-all's could contain user-provided input, which need to be greedily
+#   matched away first.
+#
+# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black
+
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 7b08b936..af7af1bc 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -38,6 +38,7 @@ def _test_output(*args):
 
 fail2banregex.output = _test_output
 
+TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 DEV_NULL = None
@@ -85,6 +86,11 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd")
 	FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf')
+	FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log')
+	FILTER_ZZZ_SSHD = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-sshd-obsolete-multiline.conf')
+
+	FILENAME_ZZZ_GEN = os.path.join(TEST_FILES_DIR, "logs", "zzz-generic-example")
+	FILTER_ZZZ_GEN = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-generic-example.conf')
 
 	def setUp(self):
 		"""Call before every test case."""
@@ -210,6 +216,39 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged("[29116]: User root not allowed because account is locked",
 			"[29116]: Received disconnect from 1.2.3.4", all=True)
 
+	def testFastSshd(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
+			"--print-all-matched",
+			Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_SSHD
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		# test failure line and all not-failure lines presents:
+		self.assertLogged(
+			"[29116]: Connection from 192.0.2.4",
+			"[29116]: User root not allowed because account is locked",
+			"[29116]: Received disconnect from 192.0.2.4", all=True)
+
+	def testMultilineSshd(self):
+		# by the way test of missing lines by multiline in `for bufLine in orgLineBuffer[int(fullBuffer):]`
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
+			"--print-all-matched", "--print-all-missed",
+			Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_ZZZ_SSHD
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		# test "failure" line presents (2nd part only, because multiline fewer precise):
+		self.assertLogged(
+			"[29116]: Received disconnect from 192.0.2.4", all=True)
+
+	def testFullGeneric(self):
+		# by the way test of ignoreregex (specified in filter file)...
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
+			Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN
+		)
+		self.assertTrue(fail2banRegex.start(args))
+
 	def _reset(self):
 		# reset global warn-counter:
 		from ..server.filter import _decode_line_warn
diff --git a/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline b/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline
new file mode 100644
index 00000000..d301ad91
--- /dev/null
+++ b/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline
@@ -0,0 +1,2 @@
+# test sshd file:
+# addFILE: "sshd"
diff --git a/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log b/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log
new file mode 100644
index 00000000..e0f6cc8a
--- /dev/null
+++ b/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log
@@ -0,0 +1,4 @@
+Apr 27 13:02:01 host sshd[29116]: Connection from 192.0.2.4 port 55555
+Apr 27 13:02:02 host sshd[29116]: User root not allowed because account is locked
+Apr 27 13:02:03 host sshd[29116]: input_userauth_request: invalid user root [preauth]
+Apr 27 13:02:04 host sshd[29116]: Received disconnect from 192.0.2.4: 11: Normal Shutdown, Thank you for playing [preauth]

From 32ac383d06b70c5a095be833548f67a7fa67ab09 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 27 Feb 2017 15:51:33 +0100
Subject: [PATCH 314/623] Update ChangeLog

---
 ChangeLog | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d720575c..83d61a55 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -10,6 +10,45 @@ ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
 -----------
 
 TODO: implementing of options resp. other tasks from PR #1346
+      documentation should be extended (new options, etc)
+      
+### Fixes
+* `filter.d/pam-generic.conf`:
+  - [grave] injection on user name to host fixed
+* `action.d/complain.conf`
+  - fixed using new tag `<ip-rev>` (sh/dash compliant now)
+
+### New Features
+* New Actions:
+
+* New Filters:
+
+### Enhancements
+* Introduced new filter option `prefregex` for pre-filtering using single regular expression (gh-1698);
+* Many times faster and fewer CPU-hungry because of parsing with `maxlines=1`, so without 
+  line buffering (scrolling of the buffer-window).
+  Combination of tags `<F-MLFID>` and `<F-NOFAIL>` can be used now to process multi-line logs
+  using single-line expressions:
+  - tag `<F-MLFID>`: used to identify resp. store failure info for groups of log-lines with the same
+    identifier (e. g. combined failure-info for the same conn-id by `<F-MLFID>(?:conn-id)</F-MLFID>`,
+    see sshd.conf for example)
+  - tag `<F-NOFAIL>`: used as mark for no-failure (helper to accumulate common failure-info, 
+    e. g. from lines that contain IP-address);
+* Several filters optimized with pre-filtering using new option `prefregex`, and multiline filter 
+  using `<F-MLFID>` + `<F-NOFAIL>` combination;
+* Exposes filter group captures in actions (non-recursive interpolation of tags `<F-...>`, 
+  see gh-1698, gh-1110)
+* Some filters extended with user name (can be used in gh-1243 to distinguish IP and user,
+  resp. to remove after success login the user-related failures only);
+* Safer, more stable and faster replaceTag interpolation (switched from cycle over all tags
+  to re.sub with callable)
+* substituteRecursiveTags optimization + moved in helpers facilities (because currently used 
+  commonly in server and in client)
+* Provides new tag `<ip-rev>` for PTR reversed representation of IP address
+
+
+ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
+-----------
 
 ### Fixes
 * [Grave] memory leak's fixed (gh-1277, gh-1234)

From b832b77e3cc0fee13253c898f2f88cb926ed596a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Feb 2017 21:42:19 +0100
Subject: [PATCH 315/623] small amendment for test-coverage; dynamical monitor
 failures test classes get proper names running in python3.x (wrong
 __qualname__)

---
 fail2ban/tests/filtertestcase.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 8558f8d3..2b35edc9 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -302,7 +302,7 @@ class BasicFilter(unittest.TestCase):
 		## test function "_tm" works correct (returns the same as slow strftime):
 		for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600):
 			tm = datetime.datetime.fromtimestamp(i).strftime("%Y-%m-%d %H:%M:%S")
-			if _tm(i) != tm:
+			if _tm(i) != tm: # pragma: no cover - never reachable
 				self.assertEqual((_tm(i), i), (tm, i))
 
 	def testWrongCharInTupleLine(self):
@@ -796,7 +796,7 @@ class CommonMonitorTestCase(unittest.TestCase):
 		"""Wait up to `delay` sec to assure that expected failure `count` reached
 		"""
 		ret = Utils.wait_for(
-			lambda: self.filter.failManager.getFailTotal() >= self._failTotal + count and self.jail.isFilled(),
+			lambda: self.filter.failManager.getFailTotal() >= (self._failTotal + count) and self.jail.isFilled(),
 			_maxWaitTime(delay))
 		self._failTotal += count
 		return ret
@@ -1034,9 +1034,10 @@ def get_monitor_failures_testcase(Filter_):
 			# total count in this test:
 			self.assertEqual(self.filter.failManager.getFailTotal(), 12)
 
-	MonitorFailures.__name__ = "MonitorFailures<%s>(%s)" \
+	cls = MonitorFailures
+	cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \
 			  % (Filter_.__name__, testclass_name) # 'tempfile')
-	return MonitorFailures
+	return cls
 
 
 def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
@@ -1216,14 +1217,16 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				fields = self.journal_fields
 				fields.update(TEST_JOURNAL_FIELDS)
 				journal.send(MESSAGE=l, **fields)
+			self.waitForTicks(1)
 			self.waitFailTotal(6, 10)
 			self.assertTrue(Utils.wait_for(lambda: len(self.jail) == 2, 10))
 			self.assertEqual(sorted([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()]), 
 				["192.0.2.1", "192.0.2.2"])
 
-	MonitorJournalFailures.__name__ = "MonitorJournalFailures<%s>(%s)" \
+	cls = MonitorJournalFailures
+	cls.__qualname__ = cls.__name__ = "MonitorJournalFailures<%s>(%s)" \
 			  % (Filter_.__name__, testclass_name)
-	return MonitorJournalFailures
+	return cls
 
 
 class GetFailures(LogCaptureTestCase):

From a0bb51ef92864ec504903d75e814c14f242204cf Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 8 Mar 2017 16:34:03 +0100
Subject: [PATCH 316/623] New tag '<ip-host>' introduced: can be used in
 actions to retrieve the host name (dns) from the IP address

---
 fail2ban/server/actions.py | 1 +
 fail2ban/server/ipdns.py   | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 6b793b8f..e0719cde 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -291,6 +291,7 @@ class Actions(JailThread, Mapping):
 		AI_DICT = {
 			"ip":				lambda self: self.__ticket.getIP(),
 			"ip-rev":		lambda self: self['ip'].getPTR(''),
+			"ip-host":	lambda self: self['ip'].getHost(),
 			"fid":			lambda self: self.__ticket.getID(),
 			"failures":	lambda self: self.__ticket.getAttempt(),
 			"time":			lambda self: self.__ticket.getTime(),
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 757cceba..8990618a 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -376,6 +376,11 @@ class IPAddr(object):
 
 		return "%s.%s" % (".".join(reversed(exploded_ip)), suffix)
 
+	def getHost(self):
+		"""Return the host name (DNS) of the provided IP address object
+		"""
+		return DNSUtils.ipToName(self.ntoa)
+
 	@property
 	def isIPv4(self):
 		"""Either the IP object is of address family AF_INET

From 59cf7611298b570cea657a9b1f30741d910cf5a6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 8 Mar 2017 16:50:21 +0100
Subject: [PATCH 317/623] Real action info instead of calling map in test
 cases, covering of the new tag '<ip-host>'; dns lookup: pre-caching within
 test cases - prevent slow dns-resolving and failures if no-network, of if
 some IP addresses will be changed later

---
 fail2ban/tests/servertestcase.py | 14 +++++++++-----
 fail2ban/tests/utils.py          |  3 +++
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 464295fd..461c6b2f 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -38,7 +38,9 @@ from ..server.server import Server
 from ..server.ipdns import IPAddr
 from ..server.jail import Jail
 from ..server.jailthread import JailThread
+from ..server.ticket import BanTicket
 from ..server.utils import Utils
+from .dummyjail import DummyJail
 from .utils import LogCaptureTestCase
 from ..helpers import getLogger, PREFER_ENC
 from .. import version
@@ -1677,7 +1679,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# complain --
 				('j-complain-abuse', 
 					'complain['
-					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' +
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s Hostname: <ip-host> - ",' +
 					  # test reverse ip:
 					  'debug=1,' +
 						# 2 logs to test grep from multiple logs:
@@ -1692,14 +1694,14 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
 						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
 						# both abuse mails should be separated with space:
-						'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+						'mail -s Hostname: test-host - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
 					),
 					'ip6-ban': (
 						# test reverse ip:
 						'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org',
 						'Lines containing failures of 2001:db8::1 (max 2)',
 						# both abuse mails should be separated with space:
-						'mail -s Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+						'mail -s Hostname: test-host - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server',
 					),
 				}),
 			)
@@ -1723,6 +1725,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 			ipv4 = IPAddr('87.142.124.10')
 			ipv6 = IPAddr('2001:db8::1');
+			dmyjail = DummyJail()
 			for jail, act, tests in testJailsActions:
 				# print(jail, jails[jail])
 				for a in jails[jail].actions:
@@ -1736,7 +1739,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					for (test, ip) in (('ip4-ban', ipv4), ('ip6-ban', ipv6)):
 						if not tests.get(test): continue
 						self.pruneLog('# === %s ===' % test)
-						ticket = _actions.CallingMap({
-							'ip': ip, 'ip-rev': lambda self: self['ip'].getPTR(''), 'failures': 100,})
+						ticket = BanTicket(ip)
+						ticket.setAttempt(100)
+						ticket = _actions.Actions.ActionInfo(ticket, dmyjail)
 						action.ban(ticket)
 						self.assertLogged(*tests[test], all=True)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 78a42d09..7fba73c9 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -273,6 +273,9 @@ def initTests(opts):
 		c.set('192.0.2.%s' % i, None)
 		c.set('198.51.100.%s' % i, None)
 		c.set('203.0.113.%s' % i, None)
+		c.set('2001:db8::%s' %i, 'test-host')
+	# some legal ips used in our test cases (prevent slow dns-resolving and failures if will be changed later):
+	c.set('87.142.124.10', 'test-host')
 	if unittest.F2B.no_network: # pragma: no cover
 		# precache all wrong dns to ip's used in test cases:
 		c = DNSUtils.CACHE_nameToIp

From 6a2c95da9542c6ad2f26eda6b1d3fd131524ec0c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 8 Mar 2017 16:45:04 +0100
Subject: [PATCH 318/623] `action.d/sendmail-geoip-lines.conf` fixed using new
 tag `<ip-host>` (dns-cache and without external command execution); changelog
 updated;

---
 ChangeLog                                 | 8 +++++++-
 config/action.d/sendmail-geoip-lines.conf | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 83d61a55..d2284473 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -17,6 +17,8 @@ TODO: implementing of options resp. other tasks from PR #1346
   - [grave] injection on user name to host fixed
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
+* `action.d/sendmail-geoip-lines.conf`
+  - fixed using new tag `<ip-host>` (without external command execution)
 
 ### New Features
 * New Actions:
@@ -44,7 +46,11 @@ TODO: implementing of options resp. other tasks from PR #1346
   to re.sub with callable)
 * substituteRecursiveTags optimization + moved in helpers facilities (because currently used 
   commonly in server and in client)
-* Provides new tag `<ip-rev>` for PTR reversed representation of IP address
+* New tags (usable in actions):
+  - `<fid>` - failure identifier (if raw resp. failures without IP address)
+  - `<ip-rev>` - PTR reversed representation of IP address
+  - `<ip-host>` - host name of the IP address
+  - `<F-...>` - interpolates to the corresponding filter group capture `...`
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index 34c3aedd..decf2c05 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -36,7 +36,7 @@ actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n
             http://whois.domaintools.com/<ip>\n\n
             Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "<ip>" | cut -d':' -f2-`
             AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "<ip>" | cut -d':' -f2-`
-            hostname: `host -t A <ip> 2>&1`\n\n
+            hostname: <ip-host>\n\n
             Lines containing failures of <ip>\n";
             %(_grep_logs)s;
             printf %%b "\n

From 62fa02241f3c2c159d2d17ad8f1bf8ab96881d3a Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 9 Mar 2017 13:31:40 +0100
Subject: [PATCH 319/623] Update jail.conf

---
 config/jail.conf | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 560136a6..75a824e2 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -206,18 +206,11 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag
 # Report ban via badips.com (uses action.d/badips.conf for reporting only)
 #
 action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
+
 # Report ban via abuseipdb.com.
 #
-# See action.d/abuseipdb.conf and
-# https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
+# See action.d/abuseipdb.conf for usage example and details.
 #
-# NOTE: These reports may include sensitive Info.
-# If you want cleaner reports that ensure no user data see the helper script at the above website.
-#
-# IMPORTANT: This action relies on a api_key being added to the above action conf,
-# and the appropriate catagories set.
-# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
-# See action.d/abuseipdb.conf for catagories
 action_abuseipdb = abuseipdb
 
 # Choose default action.  To change, just override value of 'action' with the

From b1f5ac948431cfac4089c18df016f6cd02976fa9 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 9 Mar 2017 13:33:11 +0100
Subject: [PATCH 320/623] Update abuseipdb.conf

---
 config/action.d/abuseipdb.conf | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf
index 3bcfb065..15e41fbe 100644
--- a/config/action.d/abuseipdb.conf
+++ b/config/action.d/abuseipdb.conf
@@ -3,6 +3,9 @@
 # Action to report IP address to abuseipdb.com
 # You must sign up to obtain an API key from abuseipdb.com.
 #
+# NOTE: These reports may include sensitive Info.
+# If you want cleaner reports that ensure no user data see the helper script at the below website.
+#
 # IMPORTANT:
 #
 # Reporting an IP of abuse is a serious complaint. Make sure that it is
@@ -13,6 +16,15 @@
 #     password incorrectly.
 #   * For filters that have a low likelihood of receiving human errors
 #
+# This action relies on a api_key being added to the above action conf,
+# and the appropriate categories set.
+#
+# Example, for ssh bruteforce (in section [sshd] of `jail.local`): 
+#   action = %(known/action)s
+#            %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
+#
+# See below for catagories.
+#
 # Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)
 

From a683e88a74d1b7fb6e7235d86674b30bacab2c4e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Mar 2017 20:39:09 +0100
Subject: [PATCH 321/623] samples test case factory extended with filter
 options - dict in JSON to control filter options (e. g. mode, etc.):

# filterOptions: {"mode": "aggressive"}
---
 fail2ban/tests/samplestestcase.py | 50 +++++++++++++++++++------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index b59ccd98..96e0b21c 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -49,12 +49,7 @@ class FilterSamplesRegex(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		super(FilterSamplesRegex, self).setUp()
-		self.filter = Filter(None)
-		self.filter.returnRawHost = True
-		self.filter.checkAllRegex = True
-		self.filter.checkFindTime = False
-		self.filter.active = True
-
+		self.filter = None
 		setUpMyTime()
 
 	def tearDown(self):
@@ -83,11 +78,15 @@ class FilterSamplesRegex(unittest.TestCase):
 			RE_WRONG_GREED.search('non-greedy .+? test' + RE_HOST + ' test vary catch-all .* anchored$'))
 
 
-def testSampleRegexsFactory(name, basedir):
-	def testFilter(self):
-
+	def _readFilter(self, name, basedir, opts=None):
+		self.filter = Filter(None)
+		self.filter.returnRawHost = True
+		self.filter.checkAllRegex = True
+		self.filter.checkFindTime = False
+		self.filter.active = True
+		if opts is None: opts = dict()
 		# Check filter exists
-		filterConf = FilterReader(name, "jail", {}, 
+		filterConf = FilterReader(name, "jail", opts,
 			basedir=basedir, share_config=unittest.F2B.share_config)
 		self.assertEqual(filterConf.getFile(), name)
 		self.assertEqual(filterConf.getJailName(), "jail")
@@ -113,6 +112,17 @@ def testSampleRegexsFactory(name, basedir):
 				elif opt[2] == "datepattern":
 					self.filter.setDatePattern(optval)
 
+		# test regexp contains greedy catch-all before <HOST>, that is
+		# not hard-anchored at end or has not precise sub expression after <HOST>:
+		for fr in self.filter.getFailRegex():
+			if RE_WRONG_GREED.search(fr): # pragma: no cover
+				raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before <HOST>, "
+					"that is not hard-anchored at end or has not precise sub expression after <HOST>:\n%s" %
+					(name, str(fr).replace(RE_HOST, '<HOST>')))
+
+def testSampleRegexsFactory(name, basedir):
+	def testFilter(self):
+
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
 			"No sample log file available for '%s' filter" % name)
@@ -125,22 +135,21 @@ def testSampleRegexsFactory(name, basedir):
 			logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs",
 				filename))
 
-			# test regexp contains greedy catch-all before <HOST>, that is
-			# not hard-anchored at end or has not precise sub expression after <HOST>:
-			for fr in self.filter.getFailRegex():
-				if RE_WRONG_GREED.search(fr): # pragma: no cover
-					raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before <HOST>, "
-						"that is not hard-anchored at end or has not precise sub expression after <HOST>:\n%s" %
-						(name, str(fr).replace(RE_HOST, '<HOST>')))
-
 			for line in logFile:
-				jsonREMatch = re.match("^# ?(failJSON|addFILE):(.+)$", line)
+				jsonREMatch = re.match("^#+ ?(failJSON|filterOptions|addFILE):(.+)$", line)
 				if jsonREMatch:
 					try:
 						faildata = json.loads(jsonREMatch.group(2))
+						# filterOptions - dict in JSON to control filter options (e. g. mode, etc.):
+						if jsonREMatch.group(1) == 'filterOptions':
+							self.filter = None
+							self._readFilter(name, basedir, opts=faildata)
+							continue
+						# addFILE - filename to "include" test-files should be additionally parsed:
 						if jsonREMatch.group(1) == 'addFILE':
 							filenames.append(faildata)
 							continue
+						# failJSON - faildata contains info of the failure to check it.
 					except ValueError as e:
 						raise ValueError("%s: %s:%i" %
 							(e, logFile.filename(), logFile.filelineno()))
@@ -150,6 +159,9 @@ def testSampleRegexsFactory(name, basedir):
 				else:
 					faildata = {}
 
+				if self.filter is None:
+					self._readFilter(name, basedir, opts=None)
+
 				try:
 					ret = self.filter.processLine(line)
 					if not ret:

From 7e442c5b2744fed7771d5cec440e91d85072048d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Mar 2017 20:43:53 +0100
Subject: [PATCH 322/623] filter.d/sendmail-reject.conf: - rewritten using
 `prefregex` and used MLFID-related multi-line parsing (by using tag
 `<F-MLFID>` instead of buffering with `maxlines`); - optional parameter
 `mode` introduced: normal (default), extra or aggressive (see sendmail-reject
 for regex details);

test cases extended
---
 config/filter.d/sendmail-reject.conf      | 47 +++++++++++++++--------
 fail2ban/tests/files/logs/sendmail-reject | 27 ++++++++-----
 2 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf
index 2f8fd882..0793a99b 100644
--- a/config/filter.d/sendmail-reject.conf
+++ b/config/filter.d/sendmail-reject.conf
@@ -21,30 +21,45 @@ before = common.conf
 
 _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 
-failregex = ^%(__prefix_line)s\w{14}: ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[<HOST>\]( \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
-            ^%(__prefix_line)sruleset=check_relay, arg1=(?P<dom>\S+), arg2=<HOST>, relay=((?P=dom) )?\[(\d+\.){3}\d+\]( \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^%(__prefix_line)s\w{14}: rejecting commands from (\S* )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
-            ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]: ((?i)expn|vrfy) \S+ \[rejected\]$
-            ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here$<SKIPLINES>^(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$
+prefregex = ^<F-MLFID>%(__prefix_line)s(?:\w{14}: )?</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[<HOST>\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=<HOST>, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
+            ^rejecting commands from (\S* )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
+            ^(?:\S+ )?\[<HOST>\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
+            ^<[^@]+@[^>]+>\.\.\. No such user here$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$
 
-ignoreregex =
+mdre-normal =
 
+mdre-extra = ^(?:\S+ )?\[<HOST>\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to M(?:TA|SP)(?:-\w+)?$
 
-[Init]
+mdre-aggressive = %(mdre-extra)s
+
+failregex = %(cmnfailre)s
+            <mdre-<mode>>
+
+# Parameter "mode": normal (default), extra or aggressive
+# Usage example (for jail.local):
+#   [sendmail-reject]
+#   filter = sendmail-reject[mode=extra]
+#
+mode = normal
+
+ignoreregex = 
 
-# "maxlines" is number of log lines to buffer for multi-line regex searches
-maxlines = 10
 
 # DEV NOTES:
 # 
-# Regarding the last multiline regex:
+# Regarding the multiline regex:
 #
-# There can be a nunber of non-related lines between the first and second part
-# of this regex maxlines of 10 is quite generious. Only one of the 
-# "No such user" lines needs to be matched before the line with the HOST.
+# "No such user" lines generate a failure and needs to be matched together with
+# another line with the HOST, therefore no-failure line was added as regex, that
+# contains HOST (see line with tag <F-NOFAIL>).
 #
-# Note the capture __prefix, includes both the __prefix_lines (which includes
-# the sendmail PID), but also the \w+ which the the sendmail assigned mail ID.
+# Note the capture <F-MLFID>, includes both the __prefix_lines (which includes
+# the sendmail PID), but also the `\w{14}` which the the sendmail assigned 
+# mail ID (todo: check this is necessary, possible obsolete).
 #
-# Author: Daniel Black and Fabian Wenk
+# Author: Daniel Black, Fabian Wenk and Sergey Brester aka sebres.
+# Rewritten using prefregex by Serg G. Brester.
diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject
index 70d4dde6..44f8eb92 100644
--- a/fail2ban/tests/files/logs/sendmail-reject
+++ b/fail2ban/tests/files/logs/sendmail-reject
@@ -1,3 +1,5 @@
+# normal mode # filterOptions: {"mode": "normal"}
+
 # failJSON: { "time": "2005-02-25T03:01:10", "match": true , "host": "128.68.136.133" }
 Feb 25 03:01:10 kismet sm-acceptingconnections[27713]: s1P819mk027713: ruleset=check_rcpt, arg1=<asservnew@freemailhost.ru>, relay=128-68-136-133.broadband.corbina.ru [128.68.136.133], reject=550 5.7.1 <asservnew@freemailhost.ru>... Relaying denied. Proper authentication required.
 
@@ -69,20 +71,27 @@ Feb 22 14:02:44 batman sm-mta[4030]: s1MD2hsd004030: rrcs-24-73-201-194.se.biz.r
 
 
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <arhipov@domain.com>... No such user here
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026250: <arhipov@domain.com>... No such user here
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <anatoliy@domain.com>... No such user here
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026251: <anatoliy@domain.com>... No such user here
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <artem@domain.com>... No such user here
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: <artem@domain.com>... No such user here
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <anto@domain.com>... No such user here
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: <anto@domain.com>... No such user here
+
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <anton@domain.com>... No such user here
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <davaojk25@domain.com>... No such user here
 # failJSON: { "time": "2004-11-03T11:35:30", "match": true , "host": "95.32.23.163" }
 Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: from=<davaojk25@domain.com>, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163]
 
 # failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: <anton@domain.com>... No such user here
-# Different mail ID shouldn't match
-# failJSON: { "match": false }
-Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026255: from=<davaojk25@domain.com>, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163]
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: <anton@domain.com>... No such user here
+# failJSON: { "match": false, "desc": "Different mail ID shouldn't match" }
+Nov  3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026255: from=<anton@domain.com>, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163]
+
+# filterOptions: {"mode": "extra"}
+
+# failJSON: { "time": "2005-03-06T16:55:28", "match": true , "host": "192.0.2.194", "desc": "wrong resp. non RFC compiant (ddos prelude?), MTA-mode" }
+Mar  6 16:55:28 s192-168-0-1 sm-mta[20949]: v26LtRA0020949: some-host-24.example.org [192.0.2.194] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA
+# failJSON: { "time": "2005-03-07T15:04:37", "match": true , "host": "192.0.2.195", "desc": "wrong resp. non RFC compiant (ddos prelude?), MSP-mode, (may be forged)" }
+Mar  7 15:04:37 s192-168-0-1 sm-mta[18624]: v27K4Vj8018624: some-host-24.example.org [192.0.2.195] (may be forged) did not issue MAIL/EXPN/VRFY/ETRN during connection to MSP-v4

From 0c1707afdabe4f4df60b7817f82ab9c88c629d8e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Mar 2017 22:08:03 +0100
Subject: [PATCH 323/623] filter.d/sshd.conf: - optional parameter `mode`
 rewritten: normal (default), ddos, extra or aggressive (combines all), see
 sshd for regex details);

test cases reformatted (since "filterOptions", we don't need multiple test log-files anymore);
---
 MANIFEST                                      |  2 -
 config/filter.d/sshd-aggressive.conf          | 11 ---
 config/filter.d/sshd-ddos.conf                | 17 ----
 config/filter.d/sshd.conf                     | 92 +++++++++++--------
 config/jail.conf                              | 22 ++---
 .../filter.d/zzz-sshd-obsolete-multiline.conf | 34 ++++---
 fail2ban/tests/files/logs/sshd                | 46 ++++++++++
 fail2ban/tests/files/logs/sshd-aggressive     |  3 -
 fail2ban/tests/files/logs/sshd-ddos           | 41 ---------
 9 files changed, 132 insertions(+), 136 deletions(-)
 delete mode 100644 config/filter.d/sshd-aggressive.conf
 delete mode 100644 config/filter.d/sshd-ddos.conf
 delete mode 100644 fail2ban/tests/files/logs/sshd-aggressive
 delete mode 100644 fail2ban/tests/files/logs/sshd-ddos

diff --git a/MANIFEST b/MANIFEST
index f03f185b..b726a111 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -138,7 +138,6 @@ config/filter.d/solid-pop3d.conf
 config/filter.d/squid.conf
 config/filter.d/squirrelmail.conf
 config/filter.d/sshd.conf
-config/filter.d/sshd-ddos.conf
 config/filter.d/stunnel.conf
 config/filter.d/suhosin.conf
 config/filter.d/tine20.conf
@@ -327,7 +326,6 @@ fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/squid
 fail2ban/tests/files/logs/squirrelmail
 fail2ban/tests/files/logs/sshd
-fail2ban/tests/files/logs/sshd-ddos
 fail2ban/tests/files/logs/stunnel
 fail2ban/tests/files/logs/suhosin
 fail2ban/tests/files/logs/tine20
diff --git a/config/filter.d/sshd-aggressive.conf b/config/filter.d/sshd-aggressive.conf
deleted file mode 100644
index 98175cbe..00000000
--- a/config/filter.d/sshd-aggressive.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-# Fail2Ban aggressive ssh filter for at attempted exploit
-#
-# Includes failregex of both sshd and sshd-ddos filters
-#
-[INCLUDES]
-
-before = sshd.conf
-
-[Definition]
-
-mode = %(aggressive)s
diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf
deleted file mode 100644
index 69b42069..00000000
--- a/config/filter.d/sshd-ddos.conf
+++ /dev/null
@@ -1,17 +0,0 @@
-# Fail2Ban ssh filter for at attempted exploit
-#
-# The regex here also relates to a exploit:
-#
-#  http://www.securityfocus.com/bid/17958/exploit
-#  The example code here shows the pushing of the exploit straight after
-#  reading the server version. This is where the client version string normally
-#  pushed. As such the server will read this unparsible information as
-#  "Did not receive identification string".
-
-[INCLUDES]
-
-before = sshd.conf
-
-[Definition]
-
-mode = %(ddos)s
diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 922ea193..8163aa03 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -24,45 +24,59 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)?
 __suff = (?: \[preauth\])?\s*
 __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
 
-prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
-
-mode = %(normal)s
-
-normal = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
-         ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
-         ^Failed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-         ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
-         ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
-         ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
-         ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
-         ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
-         ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
-         ^Received disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-         ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
-         ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
-         ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
-         ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
-         ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
-         ^Disconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__suff)s
-         ^<F-NOFAIL>Received disconnect</F-NOFAIL> from <HOST>: 11:
-         ^<F-NOFAIL>Connection closed</F-NOFAIL> by <HOST>%(__suff)s$
-
-ddos =   ^Did not receive identification string from <HOST>%(__suff)s$
-         ^Received disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
-         ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
-         ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$
-         ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
-         ^Read from socket failed: Connection reset by peer \[preauth\]
-
-common = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
-
-aggressive = %(normal)s
-             %(ddos)s
-
 [Definition]
 
-failregex = %(mode)s
-            %(common)s
+prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
+
+cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
+            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
+            ^Failed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
+            ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+            ^Received disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+            ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
+            ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+            ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
+            ^Disconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__suff)s
+            ^<F-NOFAIL>Received disconnect</F-NOFAIL> from <HOST>: 11:
+            ^<F-NOFAIL>Connection closed</F-NOFAIL> by <HOST>%(__suff)s$
+
+mdre-normal =
+
+mdre-ddos = ^Did not receive identification string from <HOST>%(__suff)s$
+            ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
+            ^Read from socket failed: Connection reset by peer \[preauth\]
+
+mdre-extra = ^Received disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+            ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
+            ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+
+mdre-aggressive = %(mdre-ddos)s
+                  %(mdre-extra)s
+
+cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
+
+failregex = %(cmnfailre)s
+            <mdre-<mode>>
+            %(cfooterre)s
+
+# Parameter "mode": normal (default), ddos, extra or aggressive (combines all)
+# Usage example (for jail.local):
+#   [sshd]
+#   mode = extra
+#   # or another jail (rewrite filter parameters of jail):
+#   [sshd-aggressive]
+#   filter = sshd[mode=aggressive]
+#
+mode = normal
+
+#filter = sshd[mode=aggressive]
 
 ignoreregex = 
 
@@ -79,5 +93,5 @@ datepattern = {^LN-BEG}
 #   and later catch-all's could contain user-provided input, which need to be greedily
 #   matched away first.
 #
-# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black
-
+# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black and Sergey Brester aka sebres
+# Rewritten using prefregex (and introduced "mode" parameter) by Serg G. Brester.
diff --git a/config/jail.conf b/config/jail.conf
index 3e917b34..c5440b71 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -229,17 +229,11 @@ action = %(action_)s
 
 [sshd]
 
-# To use more aggressive sshd filter (inclusive sshd-ddos failregex):
-#filter = sshd-aggressive
-port    = ssh
-logpath = %(sshd_log)s
-backend = %(sshd_backend)s
-
-
-[sshd-ddos]
-# This jail corresponds to the standard configuration in Fail2ban.
-# The mail-whois action send a notification e-mail with a whois request
-# in the body.
+# To use more aggressive sshd modes set filter parameter "mode" in jail.local:
+# normal (default), ddos, extra or aggressive (combines all).
+# See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details.
+mode    = normal
+filter  = sshd[mode=%(mode)s]
 port    = ssh
 logpath = %(sshd_log)s
 backend = %(sshd_backend)s
@@ -555,7 +549,11 @@ backend = %(syslog_backend)s
 
 
 [sendmail-reject]
-
+# To use more aggressive modes set filter parameter "mode" in jail.local:
+# normal (default), extra or aggressive
+# See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details.
+mode    = normal
+filter  = sendmail-reject[mode=%(mode)s]
 port     = smtp,465,submission
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
index 729fecd7..4f28e60f 100644
--- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
@@ -23,9 +23,9 @@ __prefix_line_sl = %(__prefix_line)s%(__pref)s
 __prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s
 __prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
 
-mode = %(normal)s
+[Definition]
 
-normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
+cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
          ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
          ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
@@ -43,18 +43,30 @@ normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .*
          ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
          ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$
 
-ddos =   ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
-         ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
-         ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
-         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
-         ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
+mdre-normal =
 
-aggressive = %(normal)s
-             %(ddos)s
+mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
+             ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
 
-[Definition]
+mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+             ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
+             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
 
-failregex = %(mode)s
+mdre-aggressive = %(mdre-ddos)s
+                  %(mdre-extra)s
+
+failregex = %(cmnfailre)s
+            <mdre-<mode>>
+
+# Parameter "mode": normal (default), ddos, extra or aggressive (combines all)
+# Usage example (for jail.local):
+#   [sshd]
+#   mode = extra
+#   # or another jail (rewrite filter parameters of jail):
+#   [sshd-aggressive]
+#   filter = sshd[mode=aggressive]
+#
+mode = normal
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index e2c5f537..b53b3d96 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -183,3 +183,49 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S
 # Match sshd auth errors on OpenSUSE systems
 # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" }
 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217  user=root
+
+# filterOptions: {"mode": "ddos"}
+
+# http://forums.powervps.com/showthread.php?t=1667
+# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" }
+Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114
+
+# gh-864(1):
+# failJSON: { "match": false }
+Nov 24 23:46:39 host sshd[32686]: SSH: Server;Ltype: Version;Remote: 127.0.0.1-1780;Protocol: 2.0;Client: libssh2_1.4.3
+# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (1)" }
+Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
+
+# gh-864(2):
+# failJSON: { "match": false }
+Nov 24 23:46:40 host sshd[32686]: SSH: Server;Ltype: Kex;Remote: 127.0.0.1-1780;Enc: aes128-ctr;MAC: hmac-sha1;Comp: none [preauth]
+# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (2)" }
+Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
+
+# gh-864(3):
+# failJSON: { "match": false }
+Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-1780;Name: root [preauth]
+# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" }
+Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
+
+# filterOptions: {"mode": "extra"}
+
+# several other cases from gh-864:
+# failJSON: { "time": "2004-11-25T01:34:12", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" }
+Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth]
+# failJSON: { "time": "2004-11-25T01:35:13", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" }
+Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth]
+# failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" }
+Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth]
+
+# gh-1545:
+# failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" }
+Nov 26 13:03:29 srv sshd[45]: Unable to negotiate with 192.0.2.1 port 55419: no matching cipher found. Their offer: aes256-cbc,rijndael-cbc@lysator.liu.se,aes192-cbc,aes128-cbc,arcfour128,arcfour,3des-cbc,none [preauth]
+
+# gh-1117:
+# failJSON: { "time": "2004-11-26T13:03:30", "match": true , "host": "192.0.2.2", "desc": "No matching key exchange method" }
+Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching key exchange method found. Their offer: diffie-hellman-group1-sha1
+# failJSON: { "match": false }
+Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
+# failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
+Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
\ No newline at end of file
diff --git a/fail2ban/tests/files/logs/sshd-aggressive b/fail2ban/tests/files/logs/sshd-aggressive
deleted file mode 100644
index 5b4d3a12..00000000
--- a/fail2ban/tests/files/logs/sshd-aggressive
+++ /dev/null
@@ -1,3 +0,0 @@
-# sshd-aggressive includes sshd and sshd-ddos failregex's:
-# addFILE: "sshd"
-# addFILE: "sshd-ddos"
\ No newline at end of file
diff --git a/fail2ban/tests/files/logs/sshd-ddos b/fail2ban/tests/files/logs/sshd-ddos
deleted file mode 100644
index f0a787a1..00000000
--- a/fail2ban/tests/files/logs/sshd-ddos
+++ /dev/null
@@ -1,41 +0,0 @@
-# http://forums.powervps.com/showthread.php?t=1667
-# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" }
-Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114
-
-# gh-864(1):
-# failJSON: { "match": false }
-Nov 24 23:46:39 host sshd[32686]: SSH: Server;Ltype: Version;Remote: 127.0.0.1-1780;Protocol: 2.0;Client: libssh2_1.4.3
-# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (1)" }
-Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
-
-# gh-864(2):
-# failJSON: { "match": false }
-Nov 24 23:46:40 host sshd[32686]: SSH: Server;Ltype: Kex;Remote: 127.0.0.1-1780;Enc: aes128-ctr;MAC: hmac-sha1;Comp: none [preauth]
-# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (2)" }
-Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
-
-# gh-864(3):
-# failJSON: { "match": false }
-Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-1780;Name: root [preauth]
-# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" }
-Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
-
-# several other cases from gh-864:
-# failJSON: { "time": "2004-11-25T01:34:12", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" }
-Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth]
-# failJSON: { "time": "2004-11-25T01:35:13", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" }
-Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth]
-# failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" }
-Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth]
-
-# gh-1545:
-# failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" }
-Nov 26 13:03:29 srv sshd[45]: Unable to negotiate with 192.0.2.1 port 55419: no matching cipher found. Their offer: aes256-cbc,rijndael-cbc@lysator.liu.se,aes192-cbc,aes128-cbc,arcfour128,arcfour,3des-cbc,none [preauth]
-
-# gh-1117:
-# failJSON: { "time": "2004-11-26T13:03:30", "match": true , "host": "192.0.2.2", "desc": "No matching key exchange method" }
-Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching key exchange method found. Their offer: diffie-hellman-group1-sha1
-# failJSON: { "match": false }
-Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
-# failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
-Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
\ No newline at end of file

From 8af7a73bfc0e0588766dfd7a74f3c5249fa8d767 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 10 Mar 2017 22:14:39 +0100
Subject: [PATCH 324/623] update ChangeLog

---
 ChangeLog | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 71e46daa..5ca9fa07 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,14 @@ TODO: implementing of options resp. other tasks from PR #1346
 ### Fixes
 * `filter.d/pam-generic.conf`:
   - [grave] injection on user name to host fixed
+* `filter.d/sshd.conf`:
+  - rewritten using `prefregex` and used MLFID-related multi-line parsing
+    (by using tag `<F-MLFID>` instead of buffering with `maxlines`);
+  - optional parameter `mode` rewritten: normal (default), ddos, extra or aggressive (combines all),
+    see sshd for regex details)
+* filter.d/sendmail-reject.conf:
+  - rewritten using `prefregex` and used MLFID-related multi-line parsing;
+  - optional parameter `mode` introduced: normal (default), extra or aggressive
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`
@@ -51,6 +59,9 @@ TODO: implementing of options resp. other tasks from PR #1346
   - `<ip-rev>` - PTR reversed representation of IP address
   - `<ip-host>` - host name of the IP address
   - `<F-...>` - interpolates to the corresponding filter group capture `...`
+* Samples test case factory extended with filter options - dict in JSON to control 
+  filter options (e. g. mode, etc.):
+  # filterOptions: {"mode": "aggressive"}
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From 6a26602ba870e9acf367b9714d23efbb0fccbd93 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Sat, 11 Mar 2017 00:06:29 +0100
Subject: [PATCH 325/623] allow to use filter options by fail2ban-regex,
 example: fail2ban-regex text.log "sshd[mode=aggressive]"

---
 fail2ban/client/fail2banregex.py        | 63 +++++++++++++++++++------
 fail2ban/client/jailreader.py           |  2 +-
 fail2ban/tests/fail2banregextestcase.py | 19 ++++++--
 3 files changed, 64 insertions(+), 20 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index e2f1f67c..539383f6 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -120,6 +120,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 				version="%prog " + version)
 
 	p.add_options([
+		Option("-c", "--config", default='/etc/fail2ban',
+			   help="set alternate config directory"),
 		Option("-d", "--datepattern",
 			   help="set custom pattern used to match date/times"),
 		Option("-e", "--encoding", default=PREFER_ENC,
@@ -271,24 +273,55 @@ class Fail2banRegex(object):
 	def readRegex(self, value, regextype):
 		assert(regextype in ('fail', 'ignore'))
 		regex = regextype + 'regex'
-		if regextype == 'fail' and (os.path.isfile(value) or os.path.isfile(value + '.conf')):
-			if os.path.basename(os.path.dirname(value)) == 'filter.d':
+		# try to check - we've case filter?[options...]?:
+		basedir = self._opts.config
+		fltFile = None
+		fltOpt = {}
+		if regextype == 'fail':
+			fltName, fltOpt = JailReader.extractOptions(value)
+			if fltName is not None:
+				if "." in fltName[~5:]:
+					tryNames = (fltName,)
+				else:
+					tryNames = (fltName, fltName + '.conf', fltName + '.local')
+				for fltFile in tryNames:
+					if not "/" in fltFile:
+						if os.path.basename(basedir) == 'filter.d':
+							fltFile = os.path.join(basedir, fltFile)
+						else:
+							fltFile = os.path.join(basedir, 'filter.d', fltFile)
+					else:
+						basedir = os.path.dirname(fltFile)
+					if os.path.isfile(fltFile):
+						break
+					fltFile = None
+		# if it is filter file:
+		if fltFile is not None:
+			if (basedir == self._opts.config
+				or os.path.basename(basedir) == 'filter.d'
+				or ("." not in fltName[~5:] and "/" not in fltName)
+			):
 				## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.):
-				basedir = os.path.dirname(os.path.dirname(value))
-				value = os.path.splitext(os.path.basename(value))[0]
-				output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) )
-				reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir)
-				if not reader.read(): # pragma: no cover
-					output( "ERROR: failed to load filter %s" % value )
-					return False
-			else: # pragma: no cover
+				if os.path.basename(basedir) == 'filter.d':
+					basedir = os.path.dirname(basedir)
+				fltName = os.path.splitext(os.path.basename(fltName))[0]
+				output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) )
+			else:
+				## foreign file - readexplicit this file and includes if possible:
+				output( "Use %11s file : %s" % (regex, fltName) )
+				basedir = None
+			if fltOpt:
+				output( "Use   filter options : %r" % fltOpt )
+			reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir)
+			if basedir is not None: # pragma: no cover
+				ret = reader.read()
+			else:
 				## foreign file - readexplicit this file and includes if possible:
-				output( "Use %11s file : %s" % (regex, value) )
-				reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config)
 				reader.setBaseDir(None)
-				if not reader.readexplicit(): 
-					output( "ERROR: failed to read %s" % value )
-					return False
+				ret = reader.readexplicit()
+			if not ret:
+				output( "ERROR: failed to load filter %s" % value )
+				return False
 			reader.getOptions(None)
 			readercommands = reader.convert()
 
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index f4adadbf..2bef2c4f 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -43,7 +43,7 @@ logSys = getLogger(__name__)
 class JailReader(ConfigReader):
 	
 	# regex, to extract list of options:
-	optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL)
+	optionCRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
 	# regex, to iterate over single option in option list, syntax:
 	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
 	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index af7af1bc..19006831 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -209,7 +209,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"-l", "notice", # put down log-level, because of too many debug-messages
 			"-v", "--verbose-date", "--print-all-matched",
-			Fail2banRegexTest.FILENAME_SSHD, Fail2banRegexTest.FILTER_SSHD
+			"-c", CONFIG_DIR,
+			Fail2banRegexTest.FILENAME_SSHD, "sshd"
 		)
 		self.assertTrue(fail2banRegex.start(args))
 		# test failure line and not-failure lines both presents:
@@ -220,7 +221,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"-l", "notice", # put down log-level, because of too many debug-messages
 			"--print-all-matched",
-			Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_SSHD
+			"-c", CONFIG_DIR,
+			Fail2banRegexTest.FILENAME_ZZZ_SSHD, "sshd.conf[mode=normal]"
 		)
 		self.assertTrue(fail2banRegex.start(args))
 		# test failure line and all not-failure lines presents:
@@ -234,7 +236,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"-l", "notice", # put down log-level, because of too many debug-messages
 			"--print-all-matched", "--print-all-missed",
-			Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_ZZZ_SSHD
+			"-c", os.path.dirname(Fail2banRegexTest.FILTER_ZZZ_SSHD),
+			Fail2banRegexTest.FILENAME_ZZZ_SSHD, os.path.basename(Fail2banRegexTest.FILTER_ZZZ_SSHD)
 		)
 		self.assertTrue(fail2banRegex.start(args))
 		# test "failure" line presents (2nd part only, because multiline fewer precise):
@@ -245,10 +248,18 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		# by the way test of ignoreregex (specified in filter file)...
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"-l", "notice", # put down log-level, because of too many debug-messages
-			Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN
+			Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN+"[mode=test]"
 		)
 		self.assertTrue(fail2banRegex.start(args))
 
+	def testWrongFilterFile(self):
+		# use test log as filter file to cover eror cases...
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-l", "notice", # put down log-level, because of too many debug-messages
+			Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILENAME_ZZZ_GEN
+		)
+		self.assertFalse(fail2banRegex.start(args))
+
 	def _reset(self):
 		# reset global warn-counter:
 		from ..server.filter import _decode_line_warn

From eb3623e90cf1db8031cb89b40f7fdb8fee05b9f5 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 12 Mar 2017 19:04:45 +0100
Subject: [PATCH 326/623] configreader.py: correct reading real relative path
 (starting with "./"); fail2ban-regex: catch read exceptions by wrong config
 files (raise exception in verbose mode only);

---
 fail2ban/client/configreader.py         |  4 ++++
 fail2ban/client/fail2banregex.py        | 17 +++++++++++------
 fail2ban/tests/fail2banregextestcase.py |  1 -
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 2caa97ce..b7da271b 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -176,6 +176,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 		if not os.path.exists(self._basedir):
 			raise ValueError("Base configuration directory %s does not exist "
 							  % self._basedir)
+		if filename.startswith("./"): # pragma: no cover
+			filename = os.path.abspath(filename)
 		basename = os.path.join(self._basedir, filename)
 		logSys.debug("Reading configs for %s under %s " , filename, self._basedir)
 		config_files = [ basename + ".conf" ]
@@ -277,6 +279,8 @@ class DefinitionInitConfigReader(ConfigReader):
 	
 	def __init__(self, file_, jailName, initOpts, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
+		if file_.startswith("./"): # pragma: no cover
+			file_ = os.path.abspath(file_)
 		self.setFile(file_)
 		self.setJailName(jailName)
 		self._initOpts = initOpts
diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 539383f6..911abd28 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -313,12 +313,17 @@ class Fail2banRegex(object):
 			if fltOpt:
 				output( "Use   filter options : %r" % fltOpt )
 			reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir)
-			if basedir is not None: # pragma: no cover
-				ret = reader.read()
-			else:
-				## foreign file - readexplicit this file and includes if possible:
-				reader.setBaseDir(None)
-				ret = reader.readexplicit()
+			ret = None
+			try:
+				if basedir is not None:
+					ret = reader.read()
+				else:
+					## foreign file - readexplicit this file and includes if possible:
+					reader.setBaseDir(None)
+					ret = reader.readexplicit()
+			except Exception as e:
+				output("Wrong config file: %s" % (str(e),))
+				if self._verbose: raise(e)
 			if not ret:
 				output( "ERROR: failed to load filter %s" % value )
 				return False
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 19006831..1bac3a5f 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -255,7 +255,6 @@ class Fail2banRegexTest(LogCaptureTestCase):
 	def testWrongFilterFile(self):
 		# use test log as filter file to cover eror cases...
 		(opts, args, fail2banRegex) = _Fail2banRegex(
-			"-l", "notice", # put down log-level, because of too many debug-messages
 			Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILENAME_ZZZ_GEN
 		)
 		self.assertFalse(fail2banRegex.start(args))

From 30b53bb2ce0c4e7e0a0af9689c297f0428747877 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Mon, 13 Mar 2017 02:07:14 +0100
Subject: [PATCH 327/623] update ChangeLog and man/fail2ban-regex.1

---
 ChangeLog            | 2 ++
 man/fail2ban-regex.1 | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5ca9fa07..5c1ba70e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -59,6 +59,8 @@ TODO: implementing of options resp. other tasks from PR #1346
   - `<ip-rev>` - PTR reversed representation of IP address
   - `<ip-host>` - host name of the IP address
   - `<F-...>` - interpolates to the corresponding filter group capture `...`
+* Allow to use filter options by `fail2ban-regex`, example:
+  fail2ban-regex text.log "sshd[mode=aggressive]"
 * Samples test case factory extended with filter options - dict in JSON to control 
   filter options (e. g. mode, etc.):
   # filterOptions: {"mode": "aggressive"}
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 44e13c86..3f31a376 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -27,6 +27,9 @@ a string representing a 'failregex'
 .TP
 filename
 path to a filter file (filter.d/sshd.conf)
+.TP
+filtername[option=value, ..., option=value]
+short path to a filter relative filter.d in configuration base (sshd[mode=aggressive])
 .SS "IGNOREREGEX:"
 .TP
 string
@@ -42,6 +45,9 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
+\fB\-c\fR CONFIGBASE, \fB\-\-config\fR=\fI\,CONFIGBASE\/\fR
+set alternate config base directory (default /etc/fail2ban)
+.TP
 \fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR
 set custom pattern used to match date/times
 .TP

From 3cba2310ffc1fd380e0411f44878551757cd620c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 16:14:06 +0100
Subject: [PATCH 328/623] Fixes debuggex URL (tag replacement) and missing line
 stat by matched lines (without time - `matched_lines_timeextracted`); Closes
 gh-1394

---
 fail2ban/client/fail2banregex.py | 33 +++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 911abd28..782513a7 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -49,14 +49,14 @@ from ..version import version
 from .jailreader import JailReader
 from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
-from ..server.failregex import RegexException
+from ..server.failregex import Regex, RegexException
 
 from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, PREFER_ENC
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
-def debuggexURL(sample, regex):
-	q = urllib.urlencode({ 're': regex.replace('<HOST>', '(?&.ipv4)'),
+def debuggexURL(sample, regex, useDns="yes"):
+	q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns),
 							'str': sample,
 							'flavor': 'python' })
 	return 'https://www.debuggex.com/?' + q
@@ -198,15 +198,17 @@ class RegexStat(object):
 class LineStats(object):
 	"""Just a convenience container for stats
 	"""
-	def __init__(self):
+	def __init__(self, opts):
 		self.tested = self.matched = 0
 		self.matched_lines = []
 		self.missed = 0
 		self.missed_lines = []
-		self.missed_lines_timeextracted = []
 		self.ignored = 0
 		self.ignored_lines = []
-		self.ignored_lines_timeextracted = []
+		if opts.debuggex:
+			self.matched_lines_timeextracted = []
+			self.missed_lines_timeextracted = []
+			self.ignored_lines_timeextracted = []
 
 	def __str__(self):
 		return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self
@@ -230,7 +232,7 @@ class Fail2banRegex(object):
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._time_elapsed = None
-		self._line_stats = LineStats()
+		self._line_stats = LineStats(opts)
 
 		if opts.maxlines:
 			self.setMaxLines(opts.maxlines)
@@ -414,9 +416,10 @@ class Fail2banRegex(object):
 				try:
 					self._line_stats.missed_lines.pop(
 						self._line_stats.missed_lines.index("".join(bufLine)))
-					self._line_stats.missed_lines_timeextracted.pop(
-						self._line_stats.missed_lines_timeextracted.index(
-							"".join(bufLine[::2])))
+					if self._debuggex:
+						self._line_stats.missed_lines_timeextracted.pop(
+							self._line_stats.missed_lines_timeextracted.index(
+								"".join(bufLine[::2])))
 				except ValueError:
 					pass
 				else:
@@ -443,19 +446,23 @@ class Fail2banRegex(object):
 				self._line_stats.ignored += 1
 				if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1):
 					self._line_stats.ignored_lines.append(line)
-					self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped)
+					if self._debuggex:
+						self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped)
 
 			if len(ret) > 0:
 				assert(not is_ignored)
 				self._line_stats.matched += 1
 				if self._print_all_matched:
 					self._line_stats.matched_lines.append(line)
+					if self._debuggex:
+						self._line_stats.matched_lines_timeextracted.append(line_datetimestripped)
 			else:
 				if not is_ignored:
 					self._line_stats.missed += 1
 					if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
 						self._line_stats.missed_lines.append(line)
-						self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
+						if self._debuggex:
+							self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
 			self._line_stats.tested += 1
 
 		self._time_elapsed = time.time() - t0
@@ -478,7 +485,7 @@ class Fail2banRegex(object):
 					for arg in [l, regexlist]:
 						ans = [ x + [y] for x in ans for y in arg ]
 					b = map(lambda a: a[0] +  ' | ' + a[1].getFailRegex() + ' |  ' + 
-						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex()), ans)
+						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans)
 					pprint_list([x.rstrip() for x in b], header)
 				else:
 					output( "%s too many to print.  Use --print-all-%s " \

From 295f7b88c92b0c67b1b1d44f0fb9eca29aeb591d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 16:21:03 +0100
Subject: [PATCH 329/623] increase coverage

---
 fail2ban/tests/fail2banregextestcase.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 1bac3a5f..0cd0e303 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -286,11 +286,12 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"-l", "notice", # put down log-level, because of too many debug-messages
 			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--debuggex", "--print-all-matched",
-			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
+			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD,
+			r"llinco[^\\]"
 		)
 		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Error decoding line')
-		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
+		self.assertLogged('Lines: 4 lines, 1 ignored, 2 matched, 1 missed')
 
 		self.assertLogged('https://')
 

From 92d83274d9af6081796568de928285e8321171e3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 18:03:02 +0100
Subject: [PATCH 330/623] fixes cache overload in the test cases (increase max
 count and max time of CACHE_ipToName - too many entries in mock-up preset,
 longer time testing)

---
 fail2ban/tests/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 7fba73c9..5f144547 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -269,7 +269,9 @@ def initTests(opts):
 
 	# precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737):
 	c = DNSUtils.CACHE_ipToName
-	for i in xrange(255):
+	# increase max count and max time (too many entries, long time testing):
+	c.setOptions(maxCount=10000, maxTime=5*60)
+	for i in xrange(256):
 		c.set('192.0.2.%s' % i, None)
 		c.set('198.51.100.%s' % i, None)
 		c.set('203.0.113.%s' % i, None)

From c1da6611ecfdc4cf86597f614b7b5f0d780b4ab1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 18:47:26 +0100
Subject: [PATCH 331/623] [BF] prevents always converting of calling map items
 in replaceTag (without direct access of item): substituteRecursiveTags:
 ignore replacing callable items from calling map - should be converted on
 demand only (by get)

---
 fail2ban/helpers.py       | 3 +++
 fail2ban/server/action.py | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 2407df1f..726631a6 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -240,6 +240,7 @@ def substituteRecursiveTags(inptags, conditional='',
 	# init:
 	ignore = set(ignore)
 	done = set()
+	calmap = hasattr(tags, "getRawItem")
 	# repeat substitution while embedded-recursive (repFlag is True)
 	while True:
 		repFlag = False
@@ -247,6 +248,8 @@ def substituteRecursiveTags(inptags, conditional='',
 		for tag in tags.iterkeys():
 			# ignore escaped or already done (or in ignore list):
 			if tag in ignore or tag in done: continue
+			# ignore replacing callable items from calling map - should be converted on demand only (by get):
+			if calmap and callable(tags.getRawItem(tag)): continue
 			value = orgval = str(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index e7a98dbb..06a499df 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -98,6 +98,13 @@ class CallingMap(MutableMapping, object):
 		except:
 			return dict(self.data, **self.storage)
 
+	def getRawItem(self, key):
+		try:
+			value = self.storage[key]
+		except KeyError:
+			value = self.data[key]
+		return value
+
 	def __getitem__(self, key):
 		try:
 			value = self.storage[key]

From 5030e3a1228265638460fcf7ffe2385dacfd5d30 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 20:45:35 +0100
Subject: [PATCH 332/623] [Important] Prohibit replacement of recursive "tags"
 in the action info resp. calling map (very bad idea to do this): - the
 calling map contains normally dynamic values only (no recursive tags); -
 recursive replacement can be vulnerable, because can contain foreign (user)
 input captured from log (will be replaced in the shell arguments);

---
 fail2ban/helpers.py              |  6 ++--
 fail2ban/server/action.py        | 47 ++++++++++++++++++--------------
 fail2ban/tests/actiontestcase.py | 14 ++++++++++
 3 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 726631a6..de26dbcd 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -240,7 +240,7 @@ def substituteRecursiveTags(inptags, conditional='',
 	# init:
 	ignore = set(ignore)
 	done = set()
-	calmap = hasattr(tags, "getRawItem")
+	noRecRepl = hasattr(tags, "getRawItem")
 	# repeat substitution while embedded-recursive (repFlag is True)
 	while True:
 		repFlag = False
@@ -249,7 +249,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			# ignore escaped or already done (or in ignore list):
 			if tag in ignore or tag in done: continue
 			# ignore replacing callable items from calling map - should be converted on demand only (by get):
-			if calmap and callable(tags.getRawItem(tag)): continue
+			if noRecRepl and callable(tags.getRawItem(tag)): continue
 			value = orgval = str(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
@@ -284,6 +284,8 @@ def substituteRecursiveTags(inptags, conditional='',
 					# constructs like <STDIN>.
 					m = tre_search(value, m.end())
 					continue
+				# if calling map - be sure we've string:
+				if noRecRepl: repl = str(repl)
 				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 06a499df..31a1a3dc 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -453,7 +453,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo, conditional='', cache=None):
+	def replaceTag(cls, query, aInfo, conditional='', cache=None, substRec=True):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -478,23 +478,29 @@ class CommandAction(ActionBase):
 			except KeyError:
 				pass
 
-		# first try get cached tags dictionary:
-		subInfo = csubkey = None
-		if cache is not None:
-			csubkey = ('subst-tags', id(aInfo), conditional)
-			try:
-				subInfo = cache[csubkey]
-			except KeyError:
-				pass
-		# interpolation of dictionary:
-		if subInfo is None:
-			subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
-		# cache if possible:
-		if csubkey is not None:
-			cache[csubkey] = subInfo
+		# **Important**: don't replace if calling map - contains dynamic values only,
+		# no recursive tags, otherwise may be vulnerable on foreign user-input:
+		noRecRepl = isinstance(aInfo, CallingMap)
+		if noRecRepl:
+			subInfo = aInfo
+		else:
+			# substitute tags recursive (and cache if possible),
+			# first try get cached tags dictionary:
+			subInfo = csubkey = None
+			if cache is not None:
+				csubkey = ('subst-tags', id(aInfo), conditional)
+				try:
+					subInfo = cache[csubkey]
+				except KeyError:
+					pass
+			# interpolation of dictionary:
+			if subInfo is None:
+				subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags)
+			# cache if possible:
+			if csubkey is not None:
+				cache[csubkey] = subInfo
 
 		# substitution callable, used by interpolation of each tag
-		repeatSubst = {0: 0}
 		def substVal(m):
 			tag = m.group(1)			# tagname from match
 			value = None
@@ -510,18 +516,17 @@ class CommandAction(ActionBase):
 				# That one needs to be escaped since its content is
 				# out of our control
 				value = cls.escapeTag(value)
-			# possible contains tags:
-			if '<' in value:
-				repeatSubst[0] = 1
+			# replacement for tag:
 			return value
 
 		# interpolation of query:
 		count = MAX_TAG_REPLACE_COUNT + 1
 		while True:
-			repeatSubst[0] = 0
 			value = TAG_CRE.sub(substVal, query)
+			# **Important**: no recursive replacement for tags from calling map (properties only):
+			if noRecRepl: break
 			# possible recursion ?
-			if not repeatSubst or value == query: break
+			if value == query or '<' not in value: break
 			query = value
 			count -= 1
 			if count <= 0:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 8834ab8d..610857d6 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -157,6 +157,20 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertEqual(substituteRecursiveTags({'A': 'A <IP<PREF>HOST> B IP<PREF> C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}),
 						 {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'})
 
+	def testSubstRec_DontTouchUnusedCallable(self):
+		cm = CallingMap(
+			A=0,
+			B=lambda self: '<A><A>',
+			C=lambda self,i=0: 5 // int(self['A']) # raise error by access
+		)
+		# should raise no exceptions:
+		self.assertEqual(self.__action.replaceTag('test=<A>', cm), "test=0")
+		# **Important**: recursive replacement of dynamic data from calling map should be prohibited,
+		# otherwise may be vulnerable on foreign user-input:
+		self.assertEqual(self.__action.replaceTag('test=<A>--<B>--<A>', cm), "test=0--<A><A>--0")
+		# should raise an exception:
+		self.assertRaises(ZeroDivisionError, lambda: self.__action.replaceTag('test=<C>', cm))
+
 	def testReplaceTag(self):
 		aInfo = {
 			'HOST': "192.0.2.0",

From ccfd1ccb2d80087841c6e9242bf913f0ff7ee41c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 13 Mar 2017 21:56:06 +0100
Subject: [PATCH 333/623] code review, increase coverage, etc.

---
 fail2ban/client/configreader.py  | 14 +-------------
 fail2ban/helpers.py              | 30 ++++++++++++++++++++++++++++++
 fail2ban/server/action.py        |  4 ++--
 fail2ban/tests/actiontestcase.py | 29 +++++++++++++++++++++++------
 4 files changed, 56 insertions(+), 21 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index b7da271b..04502504 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -29,24 +29,12 @@ import os
 from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
-from ..helpers import getLogger, substituteRecursiveTags
+from ..helpers import getLogger, _merge_dicts, substituteRecursiveTags
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
-# if sys.version_info >= (3,5):
-# 	def _merge_dicts(x, y):
-# 		return {**x, **y}
-# else:
-def _merge_dicts(x, y):
-	r = x
-	if y:
-		r = x.copy()
-		r.update(y)
-	return r
-
-
 class ConfigReader():
 	"""Generic config reader class.
 
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index de26dbcd..556ca173 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -169,6 +169,36 @@ def splitwords(s):
 		return []
 	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
 
+if sys.version_info >= (3,5):
+	eval(compile(r'''if 1:
+	def _merge_dicts(x, y):
+		"""Helper to merge dicts.
+		"""
+		if y:
+			return {**x, **y}
+		return x
+	
+	def _merge_copy_dicts(x, y):
+		"""Helper to merge dicts to guarantee a copy result (r is never x).
+		"""
+		return {**x, **y}
+	''', __file__, 'exec'))
+else:
+	def _merge_dicts(x, y):
+		"""Helper to merge dicts.
+		"""
+		r = x
+		if y:
+			r = x.copy()
+			r.update(y)
+		return r
+	def _merge_copy_dicts(x, y):
+		"""Helper to merge dicts to guarantee a copy result (r is never x).
+		"""
+		r = x.copy()
+		if y:
+			r.update(y)
+		return r
 
 #
 # Following "uni_decode" function unified python independent any to string converting
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 31a1a3dc..2a773638 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -36,7 +36,7 @@ from .failregex import mapTag2Opt
 from .ipdns import asip
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
+from ..helpers import getLogger, _merge_copy_dicts, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -148,7 +148,7 @@ class CallingMap(MutableMapping, object):
 		return len(self.data)
 
 	def copy(self): # pargma: no cover
-		return self.__class__(self.data.copy())
+		return self.__class__(_merge_copy_dicts(self.data, self.storage))
 
 
 class ActionBase(object):
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 610857d6..70562baf 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -158,18 +158,35 @@ class CommandActionTest(LogCaptureTestCase):
 						 {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'})
 
 	def testSubstRec_DontTouchUnusedCallable(self):
-		cm = CallingMap(
-			A=0,
-			B=lambda self: '<A><A>',
-			C=lambda self,i=0: 5 // int(self['A']) # raise error by access
-		)
+		cm = CallingMap({
+			'A':0,
+			'B':lambda self: '<A><A>',
+			'C':'',
+			'D':''
+		})
+		#
+		# should raise no exceptions:
+		substituteRecursiveTags(cm)
+		# add exception tag:
+		cm['C'] = lambda self,i=0: 5 // int(self['A']) # raise error by access
+		# test direct get of callable (should raise an error):
+		self.assertRaises(ZeroDivisionError, lambda: cm['C'])
+		# should raise no exceptions (tag "C" still unused):
+		substituteRecursiveTags(cm)
+		# add reference to "broken" tag:
+		cm['D'] = 'test=<C>'
+		# should raise an exception (BOOM by replacement of tag "D" recursive):
+		self.assertRaises(ZeroDivisionError, lambda: substituteRecursiveTags(cm))
+		#
 		# should raise no exceptions:
 		self.assertEqual(self.__action.replaceTag('test=<A>', cm), "test=0")
 		# **Important**: recursive replacement of dynamic data from calling map should be prohibited,
 		# otherwise may be vulnerable on foreign user-input:
 		self.assertEqual(self.__action.replaceTag('test=<A>--<B>--<A>', cm), "test=0--<A><A>--0")
-		# should raise an exception:
+		# should raise an exception (BOOM by replacement of tag "C"):
 		self.assertRaises(ZeroDivisionError, lambda: self.__action.replaceTag('test=<C>', cm))
+		# should raise no exceptions (replaces tag "D" only):
+		self.assertEqual(self.__action.replaceTag('<D>', cm), "test=<C>")
 
 	def testReplaceTag(self):
 		aInfo = {

From 482e5265d7218dfcaddefb45c43d273857145b04 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 14 Mar 2017 13:11:12 +0100
Subject: [PATCH 334/623] output execution time of each test case if verbosity
 > 2

---
 fail2ban/tests/action_d/test_badips.py     |  1 +
 fail2ban/tests/action_d/test_smtp.py       |  1 +
 fail2ban/tests/banmanagertestcase.py       |  4 ++--
 fail2ban/tests/clientbeautifiertestcase.py |  1 +
 fail2ban/tests/failmanagertestcase.py      |  3 ++-
 fail2ban/tests/filtertestcase.py           |  6 +++---
 fail2ban/tests/samplestestcase.py          |  1 +
 fail2ban/tests/servertestcase.py           |  2 +-
 fail2ban/tests/utils.py                    | 13 +++++++++++--
 9 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py
index 2f3b6723..2d08b5df 100644
--- a/fail2ban/tests/action_d/test_badips.py
+++ b/fail2ban/tests/action_d/test_badips.py
@@ -61,6 +61,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			# Must cancel timer!
 			if self.action._timer:
 				self.action._timer.cancel()
+			super(BadIPsActionTest, self).tearDown()
 
 		def testCategory(self):
 			categories = self.action.getCategories()
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index bf8e83a2..d262f73e 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -81,6 +81,7 @@ class SMTPActionTest(unittest.TestCase):
 		self.smtpd.close()
 		self._active = False
 		self._loop_thread.join()
+		super(SMTPActionTest, self).tearDown()
 
 	def _exec_and_wait(self, doaction, timeout=3, short=False):
 		if short: timeout /= 25
diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 2c9d6601..b818d617 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -38,7 +38,7 @@ class AddFailure(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-		pass
+		super(AddFailure, self).tearDown()
 
 	def testAdd(self):
 		self.assertTrue(self.__banManager.addBanTicket(self.__ticket))
@@ -147,7 +147,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-		pass
+		super(StatusExtendedCymruInfo, self).tearDown()
 
 	available = True, None
 
diff --git a/fail2ban/tests/clientbeautifiertestcase.py b/fail2ban/tests/clientbeautifiertestcase.py
index 7d20e84e..79a0ff54 100644
--- a/fail2ban/tests/clientbeautifiertestcase.py
+++ b/fail2ban/tests/clientbeautifiertestcase.py
@@ -37,6 +37,7 @@ class BeautifierTest(unittest.TestCase):
 
 	def tearDown(self):
 		""" Call after every test case """
+		super(BeautifierTest, self).tearDown()
 
 	def testGetInputCmd(self):
 		cmd = ["test"]
diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py
index ed68e2a8..18f2c545 100644
--- a/fail2ban/tests/failmanagertestcase.py
+++ b/fail2ban/tests/failmanagertestcase.py
@@ -42,7 +42,8 @@ class AddFailure(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-
+		super(AddFailure, self).tearDown()
+		
 	def _addDefItems(self):
 		self.__items = [[u'193.168.0.128', 1167605999.0],
 					    [u'193.168.0.128', 1167605999.0],
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 2b35edc9..d3217555 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -478,7 +478,7 @@ class LogFileFilterPoll(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-		pass
+		super(LogFileFilterPoll, self).tearDown()
 
 	#def testOpen(self):
 	#	self.filter.openLogFile(LogFile.FILENAME)
@@ -861,7 +861,7 @@ def get_monitor_failures_testcase(Filter_):
 			#print "D: KILLING THE FILE"
 			_killfile(self.file, self.name)
 			#time.sleep(0.2)			  # Give FS time to ack the removal
-			pass
+			super(MonitorFailures, self).tearDown()
 
 		def _sleep_4_poll(self):
 			# Since FilterPoll relies on time stamps and some
@@ -1076,7 +1076,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			if self.filter and self.filter.active:
 				self.filter.stop()
 				self.filter.join()		  # wait for the thread to terminate
-				pass
+			super(MonitorJournalFailures, self).tearDown()
 
 		def _getRuntimeJournal(self):
 			# retrieve current system journal path
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 96e0b21c..1d33cc44 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -54,6 +54,7 @@ class FilterSamplesRegex(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
+		super(FilterSamplesRegex, self).tearDown()
 		tearDownMyTime()
 
 	def testFiltersPresent(self):
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 461c6b2f..7a79b6a5 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -79,7 +79,7 @@ class TransmitterBase(unittest.TestCase):
 		"""Call after every test case."""
 		# stop jails, etc.
 		self.server.quit()
-		#super(TransmitterBase, self).tearDown()
+		super(TransmitterBase, self).tearDown()
 
 	def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False):
 		"""Process set/get commands and compare both return values 
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 5f144547..50564f15 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -531,12 +531,21 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 _org_setUp = unittest.TestCase.setUp
 def _customSetUp(self):
 	# print('=='*10, self)
-	if unittest.F2B.log_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
-		print("")
+	# so if DEBUG etc -- show them (and log it in travis)!
+	if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
+		sys.stderr.write("\n")
 		logSys.debug('='*10 + ' %s ' + '='*20, self.id())
 	_org_setUp(self)
+	if unittest.F2B.verbosity > 2: # pragma: no cover
+		self.__startTime = time.time()
+
+_org_tearDown = unittest.TestCase.tearDown
+def _customTearDown(self):
+	if unittest.F2B.verbosity > 2: # pragma: no cover
+		sys.stderr.write(" %.3fs -- " % (time.time() - self.__startTime,))
 
 unittest.TestCase.setUp = _customSetUp
+unittest.TestCase.tearDown = _customTearDown
 
 
 class LogCaptureTestCase(unittest.TestCase):

From d79267c42463a73570ba9d897da869b0b644b98b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Tue, 14 Mar 2017 20:47:31 +0100
Subject: [PATCH 335/623] Updated xarf-specification repo URL in xarf action

---
 config/action.d/xarf-login-attack.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 5274cdaf..a7ea8406 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -22,7 +22,7 @@
 # Login-Attack, Malware-Attack, Fraud (Phishing, etc.), Info DNSBL
 #
 # For details see:
-# https://github.com/abusix/xarf-specification
+# https://github.com/xarf/xarf-specification
 # http://www.x-arf.org/schemata.html
 #
 # Author: Daniel Black

From 97d417926dc1e9eef637c3ba3b45d3c48814b0e0 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Mar 2017 17:56:46 +0100
Subject: [PATCH 336/623] repairs testing of missing samples for all regex
 after filter settings (mode) changed

---
 fail2ban/server/filter.py         |  9 +++----
 fail2ban/tests/samplestestcase.py | 44 ++++++++++++++++++++-----------
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 6b782fcc..ca2dae86 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -183,15 +183,12 @@ class Filter(JailThread):
 						 "valid", index)
 
 	##
-	# Get the regular expression which matches the failure.
+	# Get the regular expressions as list.
 	#
-	# @return the regular expression
+	# @return the regular expression list
 
 	def getFailRegex(self):
-		failRegex = list()
-		for regex in self.__failRegex:
-			failRegex.append(regex.getRegex())
-		return failRegex
+		return [regex.getRegex() for regex in self.__failRegex]
 
 	##
 	# Add the regular expression which matches the failure.
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 1d33cc44..00a8f305 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -99,8 +99,8 @@ class FilterSamplesRegex(unittest.TestCase):
 				optval = opt[3]
 			elif opt[0] == 'set':
 				optval = [opt[3]]
-			else:
-				continue
+			else: # pragma: no cover - unexpected
+				self.fail('Unexpected config-token %r in stream' % (opt,))
 			for optval in optval:
 				if opt[2] == "prefregex":
 					self.filter.prefRegex = optval
@@ -115,11 +115,13 @@ class FilterSamplesRegex(unittest.TestCase):
 
 		# test regexp contains greedy catch-all before <HOST>, that is
 		# not hard-anchored at end or has not precise sub expression after <HOST>:
-		for fr in self.filter.getFailRegex():
+		regexList = self.filter.getFailRegex()
+		for fr in regexList:
 			if RE_WRONG_GREED.search(fr): # pragma: no cover
 				raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before <HOST>, "
 					"that is not hard-anchored at end or has not precise sub expression after <HOST>:\n%s" %
 					(name, str(fr).replace(RE_HOST, '<HOST>')))
+		return regexList
 
 def testSampleRegexsFactory(name, basedir):
 	def testFilter(self):
@@ -128,8 +130,17 @@ def testSampleRegexsFactory(name, basedir):
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
 			"No sample log file available for '%s' filter" % name)
 
-		regexsUsed = set()
+		regexList = None
+		regexsUsedIdx = set()
+		regexsUsedRe = set()
 		filenames = [name]
+
+		def _testMissingSamples():
+			for failRegexIndex, failRegex in enumerate(regexList):
+				self.assertTrue(
+					failRegexIndex in regexsUsedIdx or failRegex in regexsUsedRe,
+					"Regex for filter '%s' has no samples: %i: %r" %
+						(name, failRegexIndex, failRegex))
 		i = 0
 		while i < len(filenames):
 			filename = filenames[i]; i += 1;
@@ -143,25 +154,30 @@ def testSampleRegexsFactory(name, basedir):
 						faildata = json.loads(jsonREMatch.group(2))
 						# filterOptions - dict in JSON to control filter options (e. g. mode, etc.):
 						if jsonREMatch.group(1) == 'filterOptions':
+							# another filter mode - we should check previous also:
+							if self.filter is not None:
+								_testMissingSamples()
+								regexsUsedIdx = set() # clear used indices (possible overlapping by mode change)
+							# read filter with another setting:
 							self.filter = None
-							self._readFilter(name, basedir, opts=faildata)
+							regexList = self._readFilter(name, basedir, opts=faildata)
 							continue
 						# addFILE - filename to "include" test-files should be additionally parsed:
 						if jsonREMatch.group(1) == 'addFILE':
 							filenames.append(faildata)
 							continue
 						# failJSON - faildata contains info of the failure to check it.
-					except ValueError as e:
+					except ValueError as e: # pragma: no cover - we've valid json's
 						raise ValueError("%s: %s:%i" %
 							(e, logFile.filename(), logFile.filelineno()))
 					line = next(logFile)
 				elif line.startswith("#") or not line.strip():
 					continue
-				else:
+				else: # pragma: no cover - normally unreachable
 					faildata = {}
 
 				if self.filter is None:
-					self._readFilter(name, basedir, opts=None)
+					regexList = self._readFilter(name, basedir, opts=None)
 
 				try:
 					ret = self.filter.processLine(line)
@@ -174,7 +190,8 @@ def testSampleRegexsFactory(name, basedir):
 					failregex, fid, fail2banTime, fail = ret[0]
 					# Bypass no failure helpers-regexp:
 					if not faildata.get('match', False) and (fid is None or fail.get('nofail')):
-						regexsUsed.add(failregex)
+						regexsUsedIdx.add(failregex)
+						regexsUsedRe.add(regexList[failregex])
 						continue
 
 					# Check line is flagged to match
@@ -208,16 +225,13 @@ def testSampleRegexsFactory(name, basedir):
 						jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
 						fail2banTime - jsonTime) )
 
-					regexsUsed.add(failregex)
+					regexsUsedIdx.add(failregex)
+					regexsUsedRe.add(regexList[failregex])
 				except AssertionError as e: # pragma: no cover
 					raise AssertionError("%s on: %s:%i, line:\n%s" % (
 								e, logFile.filename(), logFile.filelineno(), line))
 
-		for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()):
-			self.assertTrue(
-				failRegexIndex in regexsUsed,
-				"Regex for filter '%s' has no samples: %i: %r" %
-					(name, failRegexIndex, failRegex))
+		_testMissingSamples()
 
 	return testFilter
 

From 5561423be3b2d4636f5484183c3ad470fd326d06 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Mar 2017 18:00:53 +0100
Subject: [PATCH 337/623] filter.d/sshd.conf: fixed failregex format - some
 parts are optional, new ddos more precise rule (Connection reset by with host
 entry); closes gh-1719

---
 config/filter.d/sshd.conf                                  | 7 ++++---
 .../tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 5 +++--
 fail2ban/tests/files/logs/sshd                             | 7 ++++++-
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 8163aa03..320ab59c 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -41,17 +41,18 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER>
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
             ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
-            ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
+            ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
             ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
-            ^Disconnecting: Too many authentication failures for <F-USER>.+?</F-USER>%(__suff)s
+            ^Disconnecting: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
             ^<F-NOFAIL>Received disconnect</F-NOFAIL> from <HOST>: 11:
             ^<F-NOFAIL>Connection closed</F-NOFAIL> by <HOST>%(__suff)s$
 
 mdre-normal =
 
 mdre-ddos = ^Did not receive identification string from <HOST>%(__suff)s$
+            ^Connection reset by <HOST>%(__on_port_opt)s%(__suff)s
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
-            ^Read from socket failed: Connection reset by peer \[preauth\]
+            ^Read from socket failed: Connection reset by peer%(__suff)s
 
 mdre-extra = ^Received disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
             ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
index 4f28e60f..d6eecd4b 100644
--- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
@@ -40,12 +40,13 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for
          ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$
          ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
          ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>: 11: .+%(__suff)s$
-         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
-         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$
+         ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
+         ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$
 
 mdre-normal =
 
 mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__suff)s$
+             ^%(__prefix_line_sl)sConnection reset by <HOST>%(__on_port_opt)s%(__suff)s
              ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
 
 mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index b53b3d96..fe19591c 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -168,7 +168,7 @@ Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication fa
 # failJSON: { "match": false }
 Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 on 127.0.0.1 port 22
 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" }
-Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth]
+Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures [preauth]
 
 # failJSON: { "time": "2004-11-23T21:50:37", "match": true , "host": "61.0.0.1", "desc": "New logline format as openssh 6.8 to replace prev multiline version" }
 Nov 23 21:50:37 myhost sshd[21810]: error: maximum authentication attempts exceeded for root from 61.0.0.1 port 49940 ssh2 [preauth]
@@ -208,6 +208,11 @@ Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-
 # failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" }
 Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth]
 
+# gh-1719:
+# failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" }
+Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth]
+
+
 # filterOptions: {"mode": "extra"}
 
 # several other cases from gh-864:

From 93ec9e01d402d9fcbb83f4548de1eab0bf581f13 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 17 Mar 2017 10:03:34 +0100
Subject: [PATCH 338/623] fixes a small blemish by output in beautifier;
 command "unban" returns a count of tickets that were flushed

---
 fail2ban/client/beautifier.py            | 7 ++++++-
 fail2ban/server/transmitter.py           | 8 ++++----
 fail2ban/tests/fail2banclienttestcase.py | 8 ++++++++
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index df44afbb..4d9e549f 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -89,6 +89,8 @@ class Beautifier:
 						val = " ".join(map(str, res1[1])) if isinstance(res1[1], list) else res1[1]
 						msg.append("%s %s:\t%s" % (prefix1, res1[0], val))
 				msg = "\n".join(msg)
+			elif len(inC) < 2:
+				pass # to few cmd args for below
 			elif inC[1] == "syslogsocket":
 				msg = "Current syslog socket is:\n"
 				msg += "`- " + response
@@ -110,6 +112,8 @@ class Beautifier:
 				else:
 					msg = "Current database purge age is:\n"
 					msg += "`- %iseconds" % response
+			elif len(inC) < 3:
+				pass # to few cmd args for below
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
@@ -178,7 +182,8 @@ class Beautifier:
 					msg += ", ".join(response)
 		except Exception:
 			logSys.warning("Beautifier error. Please report the error")
-			logSys.error("Beautify %r with %r failed", response, self.__inputCmd)
+			logSys.error("Beautify %r with %r failed", response, self.__inputCmd,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			msg = repr(msg) + repr(response)
 		return msg
 
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 265b9704..ad21b851 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -108,11 +108,11 @@ class Transmitter:
 			value = command[1:]
 			# if all ips:
 			if len(value) == 1 and value[0] == "--all":
-				self.__server.setUnbanIP()
-				return
+				return self.__server.setUnbanIP()
+			cnt = 0
 			for value in value:
-				self.__server.setUnbanIP(None, value)
-			return None
+				cnt += self.__server.setUnbanIP(None, value)
+			return cnt
 		elif command[0] == "echo":
 			return command[1:]
 		elif command[0] == "sleep":
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index adaf4719..683368ee 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -1075,6 +1075,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail1] Ban 192.0.2.4", all=True
 		)
 
+		# unban all (just to test command, already empty - nothing to unban):
+		self.pruneLog("[test-phase 7b]")
+		self.execSuccess(startparams,
+			"--async", "unban", "--all")
+		self.assertLogged(
+			"Flush ban list",
+			"Unbanned 0, 0 ticket(s) in 'test-jail1'", all=True)
+
 		# backend-switch (restart instead of reload):
 		self.pruneLog("[test-phase 8a]")
 		_write_jail_cfg(enabled=[1], backend="xxx-unknown-backend-zzz")

From e5c9f9ec1cfa27180ce1f0f62036fe4fd42316b3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Mar 2017 16:35:40 +0100
Subject: [PATCH 339/623] [interim commit] try to fix possible escape
 vulnerability in actions

---
 fail2ban/server/action.py        | 67 +++++++++++++++++++++-----------
 fail2ban/server/actions.py       |  1 +
 fail2ban/server/ipdns.py         |  5 +++
 fail2ban/server/utils.py         | 64 +++++++++++++++++++++---------
 fail2ban/tests/actiontestcase.py | 28 ++++++++++---
 5 files changed, 119 insertions(+), 46 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 2a773638..bdb25d27 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -453,7 +453,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo, conditional='', cache=None, substRec=True):
+	def replaceTag(cls, query, aInfo, conditional='', cache=None):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -481,9 +481,8 @@ class CommandAction(ActionBase):
 		# **Important**: don't replace if calling map - contains dynamic values only,
 		# no recursive tags, otherwise may be vulnerable on foreign user-input:
 		noRecRepl = isinstance(aInfo, CallingMap)
-		if noRecRepl:
-			subInfo = aInfo
-		else:
+		subInfo = aInfo
+		if not noRecRepl:
 			# substitute tags recursive (and cache if possible),
 			# first try get cached tags dictionary:
 			subInfo = csubkey = None
@@ -534,13 +533,50 @@ class CommandAction(ActionBase):
 					"unexpected too long replacement interpolation, "
 					"possible self referencing definitions in query: %s" % (query,))
 
-
 		# cache if possible:
 		if cache is not None:
 			cache[ckey] = value
 		#
 		return value
 
+	@classmethod
+	def replaceDynamicTags(cls, realCmd, aInfo):
+		"""Replaces dynamical tags in `query` with property values.
+
+		**Important**
+		-------------
+		Because this tags are dynamic resp. foreign (user) input:
+		  - values should be escaped
+		  - no recursive substitution (no interpolation for <a<b>>)
+		  - don't use cache
+
+		Parameters
+		----------
+		query : str
+			String with tags.
+		aInfo : dict
+			Tags(keys) and associated values for substitution in query.
+
+		Returns
+		-------
+		str
+			shell script as string or array with tags replaced (direct or as variables).
+		"""
+		realCmd = cls.replaceTag(realCmd, aInfo, conditional=False)
+		# Replace ticket options (filter capture groups) non-recursive:
+		if '<' in realCmd:
+			tickData = aInfo.get("F-*")
+			if not tickData: tickData = {}
+			def substTag(m):
+				tn = mapTag2Opt(m.groups()[0])
+				try:
+					return str(tickData[tn])
+				except KeyError:
+					return ""
+			
+			realCmd = FCUSTAG_CRE.sub(substTag, realCmd)
+		return realCmd
+
 	def _processCmd(self, cmd, aInfo=None, conditional=''):
 		"""Executes a command with preliminary checks and substitutions.
 
@@ -605,21 +641,9 @@ class CommandAction(ActionBase):
 		realCmd = self.replaceTag(cmd, self._properties, 
 			conditional=conditional, cache=self.__substCache)
 
-		# Replace dynamical tags (don't use cache here)
+		# Replace dynamical tags, important - don't cache, no recursion and auto-escape here
 		if aInfo is not None:
-			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
-			# Replace ticket options (filter capture groups) non-recursive:
-			if '<' in realCmd:
-				tickData = aInfo.get("F-*")
-				if not tickData: tickData = {}
-				def substTag(m):
-					tn = mapTag2Opt(m.groups()[0])
-					try:
-						return str(tickData[tn])
-					except KeyError:
-						return ""
-				
-				realCmd = FCUSTAG_CRE.sub(substTag, realCmd)
+			realCmd = self.replaceDynamicTags(realCmd, aInfo)
 		else:
 			realCmd = cmd
 
@@ -653,8 +677,5 @@ class CommandAction(ActionBase):
 			logSys.debug("Nothing to do")
 			return True
 
-		_cmd_lock.acquire()
-		try:
+		with _cmd_lock:
 			return Utils.executeCmd(realCmd, timeout, shell=True, output=False, **kwargs)
-		finally:
-			_cmd_lock.release()
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index e0719cde..3a85e569 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -290,6 +290,7 @@ class Actions(JailThread, Mapping):
 
 		AI_DICT = {
 			"ip":				lambda self: self.__ticket.getIP(),
+			"family":   lambda self: self['ip'].familyStr,
 			"ip-rev":		lambda self: self['ip'].getPTR(''),
 			"ip-host":	lambda self: self['ip'].getHost(),
 			"fid":			lambda self: self.__ticket.getID(),
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 8990618a..bd3b812d 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -261,6 +261,11 @@ class IPAddr(object):
 	def family(self):
 		return self._family
 
+	FAM2STR = {socket.AF_INET: 'inet4', socket.AF_INET6: 'inet6'}
+	@property
+	def familyStr(self):
+		return IPAddr.FAM2STR.get(self._family)
+
 	@property
 	def plen(self):
 		return self._plen
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index b258ae77..56428294 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -28,7 +28,7 @@ import signal
 import subprocess
 import sys
 import time
-from ..helpers import getLogger, uni_decode
+from ..helpers import getLogger, _merge_dicts, uni_decode
 
 if sys.version_info >= (3, 3):
 	import importlib.machinery
@@ -116,7 +116,22 @@ class Utils():
 		return flags
 
 	@staticmethod
-	def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, success_codes=(0,)):
+	def buildShellCmd(realCmd, varsDict):
+		# build map as array of vars and command line array:
+		varsStat = ""
+		if not isinstance(realCmd, list):
+			realCmd = [realCmd]
+		i = len(realCmd)-1
+		for k, v in varsDict.iteritems():
+			varsStat += "%s=$%s " % (k, i)
+			realCmd.append(v)
+			i += 1
+		realCmd[0] = varsStat + "\n" + realCmd[0]
+		return realCmd
+
+	@staticmethod
+	def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, 
+		success_codes=(0,), varsDict=None):
 		"""Executes a command.
 
 		Parameters
@@ -131,6 +146,8 @@ class Utils():
 		output : bool
 			If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode).
 			If False, just indication of success is returned
+		varsDict: dict
+			variables supplied to the command (or to the shell script)
 
 		Returns
 		-------
@@ -146,10 +163,18 @@ class Utils():
 		"""
 		stdout = stderr = None
 		retcode = None
-		popen = None
+		popen = env = None
+		if varsDict:
+			if shell:
+				# build map as array of vars and command line array:
+				realCmd = Utils.buildShellCmd(realCmd, varsDict)
+			else: # pragma: no cover - currently unused
+				env = _merge_dicts(os.environ, varsDict)
+		realCmdId = id(realCmd)
+		outCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd)
 		try:
 			popen = subprocess.Popen(
-				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell,
+				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env,
 				preexec_fn=os.setsid  # so that killpg does not kill our process
 			)
 			# wait with timeout for process has terminated:
@@ -158,13 +183,15 @@ class Utils():
 				def _popen_wait_end():
 					retcode = popen.poll()
 					return (True, retcode) if retcode is not None else None
-				retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL)
+				# popen.poll is fast operation so we can put down the sleep interval:
+				retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL / 100)
 				if retcode:
 					retcode = retcode[1]
 			# if timeout:
 			if retcode is None:
-				logSys.error("%s -- timed out after %s seconds." %
-					(realCmd, timeout))
+				if outCmd: outCmd(logging.ERROR); outCmd = None
+				logSys.error("%x -- timed out after %s seconds." %
+					(realCmdId, timeout))
 				pgid = os.getpgid(popen.pid)
 				# if not tree - first try to terminate and then kill, otherwise - kill (-9) only:
 				os.killpg(pgid, signal.SIGTERM) # Terminate the process
@@ -185,48 +212,49 @@ class Utils():
 				return False if not output else (False, stdout, stderr, retcode)
 
 		std_level = logging.DEBUG if retcode in success_codes else logging.ERROR
+		if std_level > logSys.getEffectiveLevel():
+			if outCmd: outCmd(std_level-1); outCmd = None
 		# if we need output (to return or to log it): 
 		if output or std_level >= logSys.getEffectiveLevel():
+
 			# if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
 			if popen.stdout:
 				try:
 					if retcode is None or retcode < 0:
 						Utils.setFBlockMode(popen.stdout, False)
 					stdout = popen.stdout.read()
-				except IOError as e:
+				except IOError as e: # pragma: no cover
 					logSys.error(" ... -- failed to read stdout %s", e)
 				if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
-					logSys.log(std_level, "%s -- stdout:", realCmd)
 					for l in stdout.splitlines():
-						logSys.log(std_level, " -- stdout: %r", uni_decode(l))
+						logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l))
 				popen.stdout.close()
 			if popen.stderr:
 				try:
 					if retcode is None or retcode < 0:
 						Utils.setFBlockMode(popen.stderr, False)
 					stderr = popen.stderr.read()
-				except IOError as e:
+				except IOError as e: # pragma: no cover
 					logSys.error(" ... -- failed to read stderr %s", e)
 				if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
-					logSys.log(std_level, "%s -- stderr:", realCmd)
 					for l in stderr.splitlines():
-						logSys.log(std_level, " -- stderr: %r", uni_decode(l))
+						logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l))
 				popen.stderr.close()
 
 		success = False
 		if retcode in success_codes:
-			logSys.debug("%-.40s -- returned successfully %i", realCmd, retcode)
+			logSys.debug("%x -- returned successfully %i", realCmdId, retcode)
 			success = True
 		elif retcode is None:
-			logSys.error("%-.40s -- unable to kill PID %i", realCmd, popen.pid)
+			logSys.error("%x -- unable to kill PID %i", realCmdId, popen.pid)
 		elif retcode < 0 or retcode > 128:
 			# dash would return negative while bash 128 + n
 			sigcode = -retcode if retcode < 0 else retcode - 128
-			logSys.error("%-.40s -- killed with %s (return code: %s)",
-				realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)
+			logSys.error("%x -- killed with %s (return code: %s)",
+				realCmdId, signame.get(sigcode, "signal %i" % sigcode), retcode)
 		else:
 			msg = _RETCODE_HINTS.get(retcode, None)
-			logSys.error("%-.40s -- returned %i", realCmd, retcode)
+			logSys.error("%x -- returned %i", realCmdId, retcode)
 			if msg:
 				logSys.info("HINT on %i: %s", retcode, msg % locals())
 		if output:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 70562baf..dc847290 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -389,6 +389,23 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertLogged('Nothing to do')
 		self.pruneLog()
 
+	def testExecuteWithVars(self):
+		self.assertTrue(self.__action.executeCmd(
+			r'''printf %b "foreign input:\n'''
+			r''' -- $f2bV_A --\n'''
+			r''' -- $f2bV_B --\n'''
+			r''' -- $(echo $f2bV_C) --''' # echo just replaces \n to test it as single line
+			r'''"''', 
+			varsDict={
+			'f2bV_A': 'I\'m a hacker; && $(echo $f2bV_B)', 
+			'f2bV_B': 'I"m very bad hacker', 
+			'f2bV_C': '`Very | very\n$(bad & worst hacker)`'
+		}))
+		self.assertLogged(r"""foreign input:""",
+			' -- I\'m a hacker; && $(echo $f2bV_B) --',
+			' -- I"m very bad hacker --',
+			' -- `Very | very $(bad & worst hacker)` --', all=True)
+
 	def testExecuteIncorrectCmd(self):
 		CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null')
 		self.assertLogged('HINT on 127: "Command not found"')
@@ -400,8 +417,9 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertFalse(CommandAction.executeCmd('sleep 30', timeout=timeout))
 		# give a test still 1 second, because system could be too busy
 		self.assertTrue(time.time() >= stime + timeout and time.time() <= stime + timeout + 1)
-		self.assertLogged('sleep 30 -- timed out after')
-		self.assertLogged('sleep 30 -- killed with SIGTERM')
+		self.assertLogged('sleep 30', ' -- timed out after', all=True)
+		self.assertLogged(' -- killed with SIGTERM', 
+		                  ' -- killed with SIGKILL')
 
 	def testExecuteTimeoutWithNastyChildren(self):
 		# temporary file for a nasty kid shell script
@@ -457,9 +475,9 @@ class CommandActionTest(LogCaptureTestCase):
 		# Verify that the process itself got killed
 		self.assertTrue(Utils.wait_for(lambda: not pid_exists(cpid), 3))
 		self.assertLogged('my pid ', 'Resource temporarily unavailable')
-		self.assertLogged('timed out')
-		self.assertLogged('killed with SIGTERM', 
-		                  'killed with SIGKILL')
+		self.assertLogged(' -- timed out')
+		self.assertLogged(' -- killed with SIGTERM', 
+		                  ' -- killed with SIGKILL')
 		os.unlink(tmpFilename)
 		os.unlink(tmpFilename + '.pid')
 

From 4f1473724bdb074f7f271a0dcc18ff8a928aaf2f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 15 Mar 2017 20:58:01 +0100
Subject: [PATCH 340/623] fixed grave vulnerability by wrong escape of tags by
 executing of shell actions

---
 fail2ban/server/action.py        | 44 +++++++++++++++++++++++++++++---
 fail2ban/server/utils.py         |  5 ++--
 fail2ban/tests/actiontestcase.py | 30 +++++++++++++++++++++-
 fail2ban/tests/servertestcase.py |  6 ++---
 4 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index bdb25d27..1c5eb0c9 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -539,6 +539,9 @@ class CommandAction(ActionBase):
 		#
 		return value
 
+	ESCAPE_CRE = re.compile(r"""[\\#&;`|*?~<>\^\(\)\[\]{}$'"\n\r]""")
+	ESCAPE_VN_CRE = re.compile(r"\W")
+
 	@classmethod
 	def replaceDynamicTags(cls, realCmd, aInfo):
 		"""Replaces dynamical tags in `query` with property values.
@@ -546,7 +549,7 @@ class CommandAction(ActionBase):
 		**Important**
 		-------------
 		Because this tags are dynamic resp. foreign (user) input:
-		  - values should be escaped
+		  - values should be escaped (using "escape" as shell variable)
 		  - no recursive substitution (no interpolation for <a<b>>)
 		  - don't use cache
 
@@ -562,19 +565,52 @@ class CommandAction(ActionBase):
 		str
 			shell script as string or array with tags replaced (direct or as variables).
 		"""
-		realCmd = cls.replaceTag(realCmd, aInfo, conditional=False)
+		# array for escaped vars:
+		varsDict = dict()
+
+		def escapeVal(tag, value):
+			# if the value should be escaped:
+			if cls.ESCAPE_CRE.search(value):
+				# That one needs to be escaped since its content is
+				# out of our control
+				tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
+				varsDict[tag] = value # add variable
+				value = '$'+tag	# replacement as variable
+			# replacement for tag:
+			return value
+
+		# substitution callable, used by interpolation of each tag
+		def substVal(m):
+			tag = m.group(1)			# tagname from match
+			try:
+				value = aInfo[tag]
+			except KeyError:
+				# fallback (no or default replacement)
+				return ADD_REPL_TAGS.get(tag, m.group())
+			value = str(value)		# assure string
+			# replacement for tag:
+			return escapeVal(tag, value)
+		
+		# Replace normally properties of aInfo non-recursive:
+		realCmd = TAG_CRE.sub(substVal, realCmd)
+
 		# Replace ticket options (filter capture groups) non-recursive:
 		if '<' in realCmd:
 			tickData = aInfo.get("F-*")
 			if not tickData: tickData = {}
 			def substTag(m):
-				tn = mapTag2Opt(m.groups()[0])
+				tag = mapTag2Opt(m.groups()[0])
 				try:
-					return str(tickData[tn])
+					value = str(tickData[tag])
 				except KeyError:
 					return ""
+				return escapeVal("F_"+tag, value)
 			
 			realCmd = FCUSTAG_CRE.sub(substTag, realCmd)
+
+		# build command corresponding "escaped" variables:
+		if varsDict:
+			realCmd = Utils.buildShellCmd(realCmd, varsDict)
 		return realCmd
 
 	def _processCmd(self, cmd, aInfo=None, conditional=''):
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 56428294..a11759b1 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -60,6 +60,7 @@ class Utils():
 	DEFAULT_SLEEP_TIME = 2
 	DEFAULT_SLEEP_INTERVAL = 0.2
 	DEFAULT_SHORT_INTERVAL = 0.001
+	DEFAULT_SHORTEST_INTERVAL = DEFAULT_SHORT_INTERVAL / 100
 
 
 	class Cache(object):
@@ -183,8 +184,8 @@ class Utils():
 				def _popen_wait_end():
 					retcode = popen.poll()
 					return (True, retcode) if retcode is not None else None
-				# popen.poll is fast operation so we can put down the sleep interval:
-				retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORT_INTERVAL / 100)
+				# popen.poll is fast operation so we can use the shortest sleep interval:
+				retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORTEST_INTERVAL)
 				if retcode:
 					retcode = retcode[1]
 			# if timeout:
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index dc847290..cbd0aaca 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -394,7 +394,7 @@ class CommandActionTest(LogCaptureTestCase):
 			r'''printf %b "foreign input:\n'''
 			r''' -- $f2bV_A --\n'''
 			r''' -- $f2bV_B --\n'''
-			r''' -- $(echo $f2bV_C) --''' # echo just replaces \n to test it as single line
+			r''' -- $(echo -n $f2bV_C) --''' # echo just replaces \n to test it as single line
 			r'''"''', 
 			varsDict={
 			'f2bV_A': 'I\'m a hacker; && $(echo $f2bV_B)', 
@@ -406,6 +406,34 @@ class CommandActionTest(LogCaptureTestCase):
 			' -- I"m very bad hacker --',
 			' -- `Very | very $(bad & worst hacker)` --', all=True)
 
+	def testExecuteReplaceEscapeWithVars(self):
+		self.__action.actionban = 'echo "** ban <ip>, reason: <reason> ...\\n<matches>"'
+		self.__action.actionunban = 'echo "** unban <ip>"'
+		self.__action.actionstop = 'echo "** stop monitoring"'
+		matches = [
+			'<actionunban>',
+			'" Hooray! #',
+			'`I\'m cool script kiddy',
+			'`I`m very cool > /here-is-the-path/to/bin/.x-attempt.sh',
+			'<actionstop>',
+		]
+		aInfo = {
+			'ip': '192.0.2.1',
+			'reason': 'hacking attempt ( he thought he knows how f2b internally works ;)',
+			'matches': '\n'.join(matches)
+		}
+		self.pruneLog()
+		self.__action.ban(aInfo)
+		self.assertLogged(
+			'** ban %s' % aInfo['ip'], aInfo['reason'], *matches, all=True)
+		self.assertNotLogged(
+			'** unban %s' % aInfo['ip'], '** stop monitoring', all=True)
+		self.pruneLog()
+		self.__action.unban(aInfo)
+		self.__action.stop()
+		self.assertLogged(
+			'** unban %s' % aInfo['ip'], '** stop monitoring', all=True)
+
 	def testExecuteIncorrectCmd(self):
 		CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null')
 		self.assertLogged('HINT on 127: "Command not found"')
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 7a79b6a5..8d1cfa62 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1679,7 +1679,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# complain --
 				('j-complain-abuse', 
 					'complain['
-					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s Hostname: <ip-host> - ",' +
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s \'Hostname: <ip-host>, family: <family>\' - ",' +
 					  # test reverse ip:
 					  'debug=1,' +
 						# 2 logs to test grep from multiple logs:
@@ -1694,14 +1694,14 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
 						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
 						# both abuse mails should be separated with space:
-						'mail -s Hostname: test-host - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+						'mail -s Hostname: test-host, family: inet4 - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
 					),
 					'ip6-ban': (
 						# test reverse ip:
 						'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org',
 						'Lines containing failures of 2001:db8::1 (max 2)',
 						# both abuse mails should be separated with space:
-						'mail -s Hostname: test-host - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+						'mail -s Hostname: test-host, family: inet6 - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server',
 					),
 				}),
 			)

From 57e9c25449e82d3224207a024d8077513a910ef8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 17 Mar 2017 22:38:49 +0100
Subject: [PATCH 341/623] bug fix in the config readers: mixing with the init
 section should affect only own init options (from init section only bypass
 default section); the situation details:   value of "_daemon" from default
 section "default" (with init section) falsely overwrites it from definition
 section "test" -   the resulting value of "_daemon" should be "test" in all 3
 resulting failregex's (as specified in test.local),   fixed and covered now;
 additionally more complex cases covered also (all filter parameters in jail
 via "%(known/...)s", dynamical interpolation across all, etc);

---
 fail2ban/client/configparserinc.py        | 16 +++++++++--
 fail2ban/client/configreader.py           | 34 +++++++++++++++--------
 fail2ban/client/jailreader.py             |  6 ++--
 fail2ban/tests/config/filter.d/test.conf  | 11 ++++++--
 fail2ban/tests/config/filter.d/test.local | 11 +++++++-
 fail2ban/tests/config/jail.conf           |  4 +--
 6 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 35fa7498..a0e02228 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -32,7 +32,7 @@ from ..helpers import getLogger
 if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
-	from configparser import ConfigParser as SafeConfigParser, \
+	from configparser import ConfigParser as SafeConfigParser, NoSectionError, \
 		BasicInterpolation
 
 	# And interpolation of __name__ was simply removed, thus we need to
@@ -60,7 +60,7 @@ if sys.version_info >= (3,2):
 				parser, option, accum, rest, section, map, depth)
 
 else: # pragma: no cover
-	from ConfigParser import SafeConfigParser
+	from ConfigParser import SafeConfigParser, NoSectionError
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -200,6 +200,18 @@ after = 1.conf
 	def get_sections(self):
 		return self._sections
 
+	def options(self, section, onlyOwn=False):
+		"""Return a list of option names for the given section name."""
+		try:
+			opts = self._sections[section]
+		except KeyError:
+			raise NoSectionError(section)
+		if not onlyOwn:
+			# mix it with defaults:
+			return set(opts.keys()) | set(self._defaults)
+		# only own option names:
+		return opts.keys()
+
 	def read(self, filenames, get_includes=True):
 		if not isinstance(filenames, list):
 			filenames = [ filenames ]
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 04502504..6e46e349 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -122,9 +122,9 @@ class ConfigReader():
 		if self._cfg is not None:
 			return self._cfg.merge_section(*args, **kwargs)
 
-	def options(self, *args):
+	def options(self, section, onlyOwn=False):
 		if self._cfg is not None:
-			return self._cfg.options(*args)
+			return self._cfg.options(section, onlyOwn)
 		return {}
 
 	def get(self, sec, opt, raw=False, vars={}):
@@ -297,23 +297,35 @@ class DefinitionInitConfigReader(ConfigReader):
 			self._create_unshared(self._file)
 		return SafeConfigParserWithIncludes.read(self._cfg, self._file)
 	
-	def getOptions(self, pOpts):
+	def getOptions(self, pOpts, all=False):
 		# overwrite static definition options with init values, supplied as
 		# direct parameters from jail-config via action[xtra1="...", xtra2=...]:
+		if not pOpts:
+			pOpts = dict()
 		if self._initOpts:
-			if not pOpts:
-				pOpts = dict()
 			pOpts = _merge_dicts(pOpts, self._initOpts)
 		self._opts = ConfigReader.getOptions(
 			self, "Definition", self._configOpts, pOpts)
 		self._pOpts = pOpts
 		if self.has_section("Init"):
-			for opt in self.options("Init"):
-				v = self.get("Init", opt)
-				if not opt.startswith('known/') and opt != '__name__':
+			# get only own options (without options from default):
+			getopt = lambda opt: self.get("Init", opt)
+			for opt in self.options("Init", onlyOwn=True):
+				if opt == '__name__': continue
+				v = None
+				if not opt.startswith('known/'):
+					if v is None: v = getopt(opt)
 					self._initOpts['known/'+opt] = v
-				if not opt in self._initOpts:
+				if opt not in self._initOpts:
+					if v is None: v = getopt(opt)
 					self._initOpts[opt] = v
+		if all and self.has_section("Definition"):
+			# merge with all definition options (and options from default),
+			# bypass already converted option (so merge only new options):
+			for opt in self.options("Definition"):
+				if opt == '__name__' or opt in self._opts: continue
+				self._opts[opt] = self.get("Definition", opt)
+
 
 	def _convert_to_boolean(self, value):
 		return value.lower() in ("1", "yes", "true", "on")
@@ -336,12 +348,12 @@ class DefinitionInitConfigReader(ConfigReader):
 
 	def getCombined(self, ignore=()):
 		combinedopts = self._opts
-		ignore = set(ignore).copy()
 		if self._initOpts:
-			combinedopts = _merge_dicts(self._opts, self._initOpts)
+			combinedopts = _merge_dicts(combinedopts, self._initOpts)
 		if not len(combinedopts):
 			return {}
 		# ignore conditional options:
+		ignore = set(ignore).copy()
 		for n in combinedopts:
 			cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
 			if cond:
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 2bef2c4f..58ae2a7f 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -139,11 +139,11 @@ class JailReader(ConfigReader):
 					filterName, self.__name, filterOpt, 
 					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
-				# merge options from filter as 'known/...':
-				self.__filter.getOptions(self.__opts)
-				ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
 				if not ret:
 					raise JailDefError("Unable to read the filter %r" % filterName)
+				# merge options from filter as 'known/...' (all options unfiltered):
+				self.__filter.getOptions(self.__opts, all=True)
+				ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
 			else:
 				self.__filter = None
 				logSys.warning("No filter set for jail %s" % self.__name)
diff --git a/fail2ban/tests/config/filter.d/test.conf b/fail2ban/tests/config/filter.d/test.conf
index f09d3467..9d08ef09 100644
--- a/fail2ban/tests/config/filter.d/test.conf
+++ b/fail2ban/tests/config/filter.d/test.conf
@@ -1,6 +1,13 @@
 #[INCLUDES]
 #before = common.conf
 
-[Definition]
-failregex = failure test 1 (filter.d/test.conf) <HOST>
+[DEFAULT]
+_daemon = default
 
+[Definition]
+where = conf
+failregex = failure <_daemon> <one> (filter.d/test.%(where)s) <HOST>
+
+[Init]
+# test parameter, should be overriden in jail by "filter=test[one=1,...]"
+one = *1*
diff --git a/fail2ban/tests/config/filter.d/test.local b/fail2ban/tests/config/filter.d/test.local
index 1b6cf55e..a954f81e 100644
--- a/fail2ban/tests/config/filter.d/test.local
+++ b/fail2ban/tests/config/filter.d/test.local
@@ -2,6 +2,15 @@
 #before = common.conf
 
 [Definition]
+# overwrite default daemon, additionally it should be accessible in jail with "%(known/_daemon)s":
+_daemon = test
+# interpolate previous regex (from test.conf) + new 2nd + dynamical substitution) of "two" an "where":
 failregex = %(known/failregex)s
-            failure test 2 (filter.d/test.local) <HOST>
+            failure %(_daemon)s <two> (filter.d/test.<where>) <HOST>
+# parameter "two" should be specified in jail by "filter=test[..., two=2]"
 
+[Init]
+# this parameter can be used in jail with "%(known/three)s":
+three = 3
+# this parameter "where" does not overwrite "where" in definition of test.conf (dynamical values only):
+where = local
\ No newline at end of file
diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf
index 659e3fd3..64c1b830 100644
--- a/fail2ban/tests/config/jail.conf
+++ b/fail2ban/tests/config/jail.conf
@@ -15,9 +15,9 @@ ignoreip =
 
 [test-known-interp]
 enabled = true
-filter = test
+filter = test[one=1,two=2]
 failregex = %(known/failregex)s
-            failure test 3 (jail.local) <HOST>
+            failure %(known/_daemon)s %(known/three)s (jail.local) <HOST>
 
 [missinglogfiles]
 enabled = true

From 32f3c1dbf3fa3c0e3e10dc7b3d8fe864bf67289d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 20 Mar 2017 13:34:42 +0100
Subject: [PATCH 342/623] test coverage

---
 fail2ban/client/configreader.py        | 39 +++++++++++++++-----------
 fail2ban/server/utils.py               |  2 +-
 fail2ban/tests/clientreadertestcase.py | 15 ++++++++--
 3 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 6e46e349..965a7a37 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -109,33 +109,40 @@ class ConfigReader():
 			self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs)
 
 	def sections(self):
-		if self._cfg is not None:
+		try:
 			return self._cfg.sections()
-		return []
+		except AttributeError:
+			return []
 
 	def has_section(self, sec):
-		if self._cfg is not None:
+		try:
 			return self._cfg.has_section(sec)
-		return False
-
-	def merge_section(self, *args, **kwargs):
-		if self._cfg is not None:
-			return self._cfg.merge_section(*args, **kwargs)
+		except AttributeError:
+			return False
 
+	def merge_section(self, section, *args, **kwargs):
+		try:
+			return self._cfg.merge_section(section, *args, **kwargs)
+		except AttributeError:
+			raise NoSectionError(section)
+	
 	def options(self, section, onlyOwn=False):
-		if self._cfg is not None:
+		try:
 			return self._cfg.options(section, onlyOwn)
-		return {}
+		except AttributeError:
+			raise NoSectionError(section)
 
 	def get(self, sec, opt, raw=False, vars={}):
-		if self._cfg is not None:
+		try:
 			return self._cfg.get(sec, opt, raw=raw, vars=vars)
-		return None
+		except AttributeError:
+			raise NoSectionError(sec)
 
-	def getOptions(self, *args, **kwargs):
-		if self._cfg is not None:
-			return self._cfg.getOptions(*args, **kwargs)
-		return {}
+	def getOptions(self, section, *args, **kwargs):
+		try:
+			return self._cfg.getOptions(section, *args, **kwargs)
+		except AttributeError:
+			raise NoSectionError(section)
 
 
 class ConfigReaderUnshared(SafeConfigParserWithIncludes):
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index a11759b1..bb6812c7 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -319,7 +319,7 @@ class Utils():
 				return e.errno == errno.EPERM
 			else:
 				return True
-	else:
+	else: # pragma : no cover (no windows currently supported)
 		@staticmethod
 		def pid_exists(pid):
 			import ctypes
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 37add795..bfa68e03 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -28,7 +28,7 @@ import re
 import shutil
 import tempfile
 import unittest
-from ..client.configreader import ConfigReader, ConfigReaderUnshared
+from ..client.configreader import ConfigReader, ConfigReaderUnshared, NoSectionError
 from ..client import configparserinc
 from ..client.jailreader import JailReader
 from ..client.filterreader import FilterReader
@@ -317,7 +317,17 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2)
 		self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), [])
 
-		
+	def testCommonFunction(self):
+		c = ConfigReader(share_config={})
+		# test common functionalities (no shared, without read of config):
+		self.assertEqual(c.sections(), [])
+		self.assertFalse(c.has_section('test'))
+		self.assertRaises(NoSectionError, c.merge_section, 'test', {})
+		self.assertRaises(NoSectionError, c.options, 'test')
+		self.assertRaises(NoSectionError, c.get, 'test', 'any')
+		self.assertRaises(NoSectionError, c.getOptions, 'test', {})
+
+
 class FilterReaderTest(unittest.TestCase):
 
 	def __init__(self, *args, **kwargs):
@@ -712,6 +722,7 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertEqual(opts['socket'], '/var/run/fail2ban/fail2ban.sock')
 			self.assertEqual(opts['pidfile'], '/var/run/fail2ban/fail2ban.pid')
 
+			configurator.readAll()
 			configurator.getOptions()
 			configurator.convertToProtocol()
 			commands = configurator.getConfigStream()

From f13fac5ae9a4e219d88d1d01f5d7af46e1ac5e46 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Mar 2017 00:15:57 +0100
Subject: [PATCH 343/623] amend to 5561423be3b2d4636f5484183c3ad470fd326d06:
 fixed incorrect failure counting despite the `<F-NOFAIL>` marked regex;
 extra: introduced new tag `<F-MLFFORGET>` as mark to forget current
 multi-line MLFID (e. g. connection closed); Closes gh-1727

---
 config/filter.d/sshd.conf      | 14 +++++++-------
 fail2ban/server/filter.py      | 32 ++++++++++++++++++++++----------
 fail2ban/tests/files/logs/sshd |  5 +++++
 3 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 320ab59c..95915fcc 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -37,24 +37,24 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER>
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
             ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
-            ^Received disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
+            ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
             ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
             ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
             ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
-            ^Disconnecting: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
-            ^<F-NOFAIL>Received disconnect</F-NOFAIL> from <HOST>: 11:
-            ^<F-NOFAIL>Connection closed</F-NOFAIL> by <HOST>%(__suff)s$
+            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
+            ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>: 11:
+            ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by <HOST>%(__suff)s$
 
 mdre-normal =
 
 mdre-ddos = ^Did not receive identification string from <HOST>%(__suff)s$
-            ^Connection reset by <HOST>%(__on_port_opt)s%(__suff)s
+            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>%(__on_port_opt)s%(__suff)s
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
-            ^Read from socket failed: Connection reset by peer%(__suff)s
+            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer%(__suff)s
 
-mdre-extra = ^Received disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
             ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
             ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$
 
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index ca2dae86..cbdb4857 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -554,20 +554,29 @@ class Filter(JailThread):
 			mlfidGroups = mlfidFail[1]
 			# if current line not failure, but previous was failure:
 			if fail.get('nofail') and not mlfidGroups.get('nofail'):
-				del fail['nofail'] # remove nofail flag - was already market as failure
+				del fail['nofail'] # remove nofail flag - completed with fid (host, ip)
 				self.mlfidCache.unset(mlfid) # remove cache entry
 			# if current line is failure, but previous was not:
 			elif not fail.get('nofail') and mlfidGroups.get('nofail'):
-				del mlfidGroups['nofail'] # remove nofail flag
+				del mlfidGroups['nofail'] # remove nofail flag - completed as failure
 				self.mlfidCache.unset(mlfid) # remove cache entry
+			else:
+				# cache this line info (if not forget):
+				if not fail.get('mlfforget'):
+					mlfidFail = [self.__lastDate, fail]
+					self.mlfidCache.set(mlfid, mlfidFail)
+				else:
+					self.mlfidCache.unset(mlfid) # remove cache entry
+				return fail
 			fail2 = mlfidGroups.copy()
 			fail2.update(fail)
 			fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
 			fail = fail2
-		elif fail.get('nofail'):
-			fail["matches"] = failRegex.getMatchedTupleLines()
+		elif not fail.get('mlfforget'):
 			mlfidFail = [self.__lastDate, fail]
 			self.mlfidCache.set(mlfid, mlfidFail)
+			if fail.get('nofail'):
+				fail["matches"] = failRegex.getMatchedTupleLines()
 		return fail
 
 
@@ -683,6 +692,11 @@ class Filter(JailThread):
 				mlfid = fail.get('mlfid')
 				if mlfid is not None:
 					fail = self._mergeFailure(mlfid, fail, failRegex)
+					# bypass if no-failure case:
+					if fail.get('nofail'):
+						logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure",
+							mlfid, failRegexIndex)
+						if not self.checkAllRegex: return failList
 				else:
 					# matched lines:
 					fail["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
@@ -702,18 +716,16 @@ class Filter(JailThread):
 					host = fail.get('dns')
 					if host is None:
 						# first try to check we have mlfid case (cache connection id):
-						if fid is None:
-							if mlfid:
-								fail = self._mergeFailure(mlfid, fail, failRegex)
-							else:
+						if fid is None and mlfid is None:
 								# if no failure-id also (obscure case, wrong regex), throw error inside getFailID:
 								fid = failRegex.getFailID()
 						host = fid
 						cidr = IPAddr.CIDR_RAW
 				# if mlfid case (not failure):
 				if host is None:
-					if not self.checkAllRegex: # or fail.get('nofail'):
-					 	return failList
+					logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier",
+						mlfid, failRegexIndex)
+					if not self.checkAllRegex: return failList
 					ips = [None]
 				# if raw - add single ip or failure-id,
 				# otherwise expand host to multiple ips using dns (or ignore it if not valid):
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index fe19591c..6f9a1468 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -113,6 +113,11 @@ May 27 00:16:33 host sshd[2364]: Received disconnect from 198.51.100.76: 11: Bye
 # failJSON: { "time": "2004-09-29T16:28:02", "match": true , "host": "127.0.0.1" }
 Sep 29 16:28:02 spaceman sshd[16699]: Failed password for dan from 127.0.0.1 port 45416 ssh1
 
+# failJSON: { "match": false, "desc": "no failure, just cache mlfid (conn-id)" }
+Sep 29 16:28:05 localhost sshd[16700]: Connection from 192.0.2.5
+# failJSON: { "match": false, "desc": "no failure, just covering mlfid (conn-id) forget" }
+Sep 29 16:28:05 localhost sshd[16700]: Connection closed by 192.0.2.5 [preauth]
+
 # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" }
 Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: RSA 8c:e3:aa:0f:64:51:02:f7:14:79:89:3f:65:84:7c:30, client user "dan", client host "localhost.localdomain"
 

From 1971fd4bd3eda0bfe91ee688ee41906af93043fc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Mar 2017 00:30:40 +0100
Subject: [PATCH 344/623] don't remove MLFID from cache (can recognize multiple
 attempt within the same connection)

---
 fail2ban/client/jailreader.py     |  4 ++--
 fail2ban/server/filter.py         | 26 +++++++++++---------------
 fail2ban/tests/files/logs/sshd    |  4 +++-
 fail2ban/tests/samplestestcase.py |  6 +++---
 4 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 2bef2c4f..bdb3564f 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -220,8 +220,8 @@ class JailReader(ConfigReader):
 		if self.__filter:
 			stream.extend(self.__filter.convert())
 		for opt, value in self.__opts.iteritems():
-			if opt == "logpath" and	\
-					not self.__opts.get('backend', None).startswith("systemd"):
+			if opt == "logpath":
+				if self.__opts.get('backend', None).startswith("systemd"): continue
 				found_files = 0
 				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index cbdb4857..76787cc7 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -550,26 +550,22 @@ class Filter(JailThread):
 
 	def _mergeFailure(self, mlfid, fail, failRegex):
 		mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
+		# if multi-line failure id (connection id) known:
 		if mlfidFail:
 			mlfidGroups = mlfidFail[1]
-			# if current line not failure, but previous was failure:
-			if fail.get('nofail') and not mlfidGroups.get('nofail'):
-				del fail['nofail'] # remove nofail flag - completed with fid (host, ip)
-				self.mlfidCache.unset(mlfid) # remove cache entry
-			# if current line is failure, but previous was not:
-			elif not fail.get('nofail') and mlfidGroups.get('nofail'):
-				del mlfidGroups['nofail'] # remove nofail flag - completed as failure
-				self.mlfidCache.unset(mlfid) # remove cache entry
+			# update - if not forget (disconnect/reset):
+			if not fail.get('mlfforget'):
+				mlfidGroups.update(fail)
 			else:
-				# cache this line info (if not forget):
-				if not fail.get('mlfforget'):
-					mlfidFail = [self.__lastDate, fail]
-					self.mlfidCache.set(mlfid, mlfidFail)
-				else:
-					self.mlfidCache.unset(mlfid) # remove cache entry
-				return fail
+				self.mlfidCache.unset(mlfid) # remove cached entry
+			# merge with previous info:
 			fail2 = mlfidGroups.copy()
 			fail2.update(fail)
+			if not fail.get('nofail'): # be sure we've correct current state
+				try:
+					del fail2['nofail']
+				except KeyError:
+					pass
 			fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
 			fail = fail2
 		elif not fail.get('mlfforget'):
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index 6f9a1468..f465c9a7 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -238,4 +238,6 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554
 # failJSON: { "match": false }
 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
-Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
\ No newline at end of file
+Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
+# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" }
+Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
\ No newline at end of file
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 00a8f305..5c45e729 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -200,13 +200,13 @@ def testSampleRegexsFactory(name, basedir):
 					self.assertEqual(len(ret), 1,
 						"Multiple regexs matched %r" % (map(lambda x: x[0], ret)))
 
-					# Fallback for backwards compatibility (previously no fid, was host only):
-					if faildata.get("host", None) is not None and fail.get("host", None) is None:
-						fail["host"] = fid
 					# Verify match captures (at least fid/host) and timestamp as expected
 					for k, v in faildata.iteritems():
 						if k not in ("time", "match", "desc"):
 							fv = fail.get(k, None)
+							# Fallback for backwards compatibility (previously no fid, was host only):
+							if k == "host" and fv is None:
+								fv = fid
 							self.assertEqual(fv, v)
 
 					t = faildata.get("time", None)

From b6886f2e519829dbf6124eb64e7887f3b6d6f171 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Mar 2017 09:42:27 +0100
Subject: [PATCH 345/623] SampleRegexsFactory extended with optional filter
 constraint, if testing the same log-file with multiple filters (no
 possibility to match by the old sshd-filter 'zzz-sshd-obsolete-multiline')

---
 fail2ban/tests/files/logs/sshd    | 4 ++--
 fail2ban/tests/samplestestcase.py | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index f465c9a7..fb3defea 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -239,5 +239,5 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554
 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
 Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
-# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" }
-Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
\ No newline at end of file
+# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" }
+Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 5c45e729..0ba11c2e 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -182,6 +182,9 @@ def testSampleRegexsFactory(name, basedir):
 				try:
 					ret = self.filter.processLine(line)
 					if not ret:
+						# Bypass if filter constraint specified:
+						if faildata.get('filter') and name != faildata.get('filter'):
+							continue
 						# Check line is flagged as none match
 						self.assertFalse(faildata.get('match', True),
 							"Line not matched when should have")
@@ -202,7 +205,7 @@ def testSampleRegexsFactory(name, basedir):
 
 					# Verify match captures (at least fid/host) and timestamp as expected
 					for k, v in faildata.iteritems():
-						if k not in ("time", "match", "desc"):
+						if k not in ("time", "match", "desc", "filter"):
 							fv = fail.get(k, None)
 							# Fallback for backwards compatibility (previously no fid, was host only):
 							if k == "host" and fv is None:

From 43d2cae8dae53eb51d3841450de6b0a03a2e1218 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Mar 2017 10:17:16 +0100
Subject: [PATCH 346/623] small amend that correct log trace output by forget
 MLFID (outputs the reason why it was forgotten - close, disconnect, etc.)

---
 fail2ban/server/filter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 76787cc7..72bf47d8 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -690,8 +690,8 @@ class Filter(JailThread):
 					fail = self._mergeFailure(mlfid, fail, failRegex)
 					# bypass if no-failure case:
 					if fail.get('nofail'):
-						logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure",
-							mlfid, failRegexIndex)
+						logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
+							mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure"))
 						if not self.checkAllRegex: return failList
 				else:
 					# matched lines:
@@ -719,8 +719,8 @@ class Filter(JailThread):
 						cidr = IPAddr.CIDR_RAW
 				# if mlfid case (not failure):
 				if host is None:
-					logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier",
-						mlfid, failRegexIndex)
+					logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
+						mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier"))
 					if not self.checkAllRegex: return failList
 					ips = [None]
 				# if raw - add single ip or failure-id,

From 7a03c964c29fe3f13c3993d98ff77b6e958885f5 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 21 Mar 2017 14:04:18 +0100
Subject: [PATCH 347/623] Update ChangeLog

---
 ChangeLog | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 5c1ba70e..7629fb2f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -41,9 +41,13 @@ TODO: implementing of options resp. other tasks from PR #1346
   using single-line expressions:
   - tag `<F-MLFID>`: used to identify resp. store failure info for groups of log-lines with the same
     identifier (e. g. combined failure-info for the same conn-id by `<F-MLFID>(?:conn-id)</F-MLFID>`,
-    see sshd.conf for example)
+    see sshd.conf for example);
+  - tag `<F-MLFFORGET>`: can be used as mark to forget current multi-line MLFID (e. g. by connection 
+    closed, reset or disconnect etc);
   - tag `<F-NOFAIL>`: used as mark for no-failure (helper to accumulate common failure-info, 
     e. g. from lines that contain IP-address);
+  Opposite to obsolete multi-line parsing (using buffering with `maxlines`) it is more precise and 
+  can recognize multiple failure attempts within the same connection (MLFID).
 * Several filters optimized with pre-filtering using new option `prefregex`, and multiline filter 
   using `<F-MLFID>` + `<F-NOFAIL>` combination;
 * Exposes filter group captures in actions (non-recursive interpolation of tags `<F-...>`, 

From 6ba0546824a93bf5cbe445a6ca575306c1ddddc8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 21 Mar 2017 14:53:33 +0100
Subject: [PATCH 348/623] code review and inline docu

---
 fail2ban/client/configparserinc.py |  9 ++++++---
 fail2ban/client/configreader.py    | 10 +++++++---
 fail2ban/server/utils.py           | 16 +++++++++++++---
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index a0e02228..6de513cd 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -200,13 +200,16 @@ after = 1.conf
 	def get_sections(self):
 		return self._sections
 
-	def options(self, section, onlyOwn=False):
-		"""Return a list of option names for the given section name."""
+	def options(self, section, withDefault=True):
+		"""Return a list of option names for the given section name.
+
+		Parameter `withDefault` controls the include of names from section `[DEFAULT]`
+		"""
 		try:
 			opts = self._sections[section]
 		except KeyError:
 			raise NoSectionError(section)
-		if not onlyOwn:
+		if withDefault:
 			# mix it with defaults:
 			return set(opts.keys()) | set(self._defaults)
 		# only own option names:
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 965a7a37..bbc18384 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -126,9 +126,13 @@ class ConfigReader():
 		except AttributeError:
 			raise NoSectionError(section)
 	
-	def options(self, section, onlyOwn=False):
+	def options(self, section, withDefault=False):
+		"""Return a list of option names for the given section name.
+
+		Parameter `withDefault` controls the include of names from section `[DEFAULT]`
+		"""
 		try:
-			return self._cfg.options(section, onlyOwn)
+			return self._cfg.options(section, withDefault)
 		except AttributeError:
 			raise NoSectionError(section)
 
@@ -317,7 +321,7 @@ class DefinitionInitConfigReader(ConfigReader):
 		if self.has_section("Init"):
 			# get only own options (without options from default):
 			getopt = lambda opt: self.get("Init", opt)
-			for opt in self.options("Init", onlyOwn=True):
+			for opt in self.options("Init", withDefault=False):
 				if opt == '__name__': continue
 				v = None
 				if not opt.startswith('known/'):
diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index bb6812c7..58363e76 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -118,6 +118,15 @@ class Utils():
 
 	@staticmethod
 	def buildShellCmd(realCmd, varsDict):
+		"""Generates new shell command as array, contains map as variables to
+		arguments statement (varsStat), the command (realCmd) used this variables and
+		the list of the arguments, mapped from varsDict
+
+		Example:
+			buildShellCmd('echo "V2: $v2, V1: $v1"', {"v1": "val 1", "v2": "val 2", "vUnused": "unused var"})
+		returns:
+			['v1=$0 v2=$1 vUnused=$2 \necho "V2: $v2, V1: $v1"', 'val 1', 'val 2', 'unused var']
+		"""
 		# build map as array of vars and command line array:
 		varsStat = ""
 		if not isinstance(realCmd, list):
@@ -172,7 +181,7 @@ class Utils():
 			else: # pragma: no cover - currently unused
 				env = _merge_dicts(os.environ, varsDict)
 		realCmdId = id(realCmd)
-		outCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd)
+		logCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd)
 		try:
 			popen = subprocess.Popen(
 				realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env,
@@ -190,7 +199,7 @@ class Utils():
 					retcode = retcode[1]
 			# if timeout:
 			if retcode is None:
-				if outCmd: outCmd(logging.ERROR); outCmd = None
+				if logCmd: logCmd(logging.ERROR); logCmd = None
 				logSys.error("%x -- timed out after %s seconds." %
 					(realCmdId, timeout))
 				pgid = os.getpgid(popen.pid)
@@ -207,6 +216,7 @@ class Utils():
 				if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover
 					retcode = signal.SIGKILL
 		except OSError as e:
+			if logCmd: logCmd(logging.ERROR); logCmd = None
 			stderr = "%s -- failed with %s" % (realCmd, e)
 			logSys.error(stderr)
 			if not popen:
@@ -214,7 +224,7 @@ class Utils():
 
 		std_level = logging.DEBUG if retcode in success_codes else logging.ERROR
 		if std_level > logSys.getEffectiveLevel():
-			if outCmd: outCmd(std_level-1); outCmd = None
+			if logCmd: logCmd(std_level-1); logCmd = None
 		# if we need output (to return or to log it): 
 		if output or std_level >= logSys.getEffectiveLevel():
 

From 5e93bf9bd3ec66707cb961c851a0ce448be3a02b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Mar 2017 15:52:31 +0100
Subject: [PATCH 349/623] Introduced new option "ignoreself", specifies whether
 the local resp. own IP addresses should be ignored (default is true).
 Fail2ban will not ban a host which matches such addresses.

Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS resp. IPs of the host self.
---
 config/jail.conf                 | 12 +++++++----
 fail2ban/client/jailreader.py    |  1 +
 fail2ban/protocol.py             |  2 ++
 fail2ban/server/filter.py        | 18 ++++++++++++++++
 fail2ban/server/ipdns.py         | 36 ++++++++++++++++++++++++++++++++
 fail2ban/server/server.py        |  6 ++++++
 fail2ban/server/transmitter.py   |  6 ++++++
 fail2ban/tests/filtertestcase.py | 11 ++++++++++
 fail2ban/tests/servertestcase.py | 10 +++++++++
 9 files changed, 98 insertions(+), 4 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index c5440b71..7e5cc9b7 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -44,10 +44,14 @@ before = paths-debian.conf
 # MISCELLANEOUS OPTIONS
 #
 
-# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
-# ban a host which matches an address in this list. Several addresses can be
-# defined using space (and/or comma) separator.
-ignoreip = 127.0.0.1/8 ::1
+# "ignorself" specifies whether the local resp. own IP addresses should be ignored
+# (default is true). Fail2ban will not ban a host which matches such addresses.
+#ignorself = true
+
+# "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban
+# will not ban a host which matches an address in this list. Several addresses
+# can be defined using space (and/or comma) separator.
+#ignoreip = 127.0.0.1/8 ::1
 
 # External command that will take an tagged arguments to ignore, e.g. <ip>,
 # and return true if the IP is to be ignored. False otherwise.
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 7f69155f..ca092990 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -110,6 +110,7 @@ class JailReader(ConfigReader):
 				["string", "failregex", None],
 				["string", "ignoreregex", None],
 				["string", "ignorecommand", None],
+				["bool",   "ignoreself", None],
 				["string", "ignoreip", None],
 				["string", "filter", ""],
 				["string", "datepattern", None],
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index d1c33d88..3625ec01 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -81,6 +81,7 @@ protocol = [
 ["status <JAIL> [FLAVOR]", "gets the current status of <JAIL>, with optional flavor or extended info"],
 ['', "JAIL CONFIGURATION", ""],
 ["set <JAIL> idle on|off", "sets the idle state of <JAIL>"], 
+["set <JAIL> ignoreself true|false", "allows the ignoring of own IP addresses"], 
 ["set <JAIL> addignoreip <IP>", "adds <IP> to the ignore list of <JAIL>"], 
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
 ["set <JAIL> addlogpath <FILE> ['tail']", "adds <FILE> to the monitoring list of <JAIL>, optionally starting at the 'tail' of the file (default 'head')."], 
@@ -117,6 +118,7 @@ protocol = [
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
 ["get <JAIL> logencoding", "gets the encoding of the log files for <JAIL>"],
 ["get <JAIL> journalmatch", "gets the journal filter match for <JAIL>"],
+["get <JAIL> ignoreself", "gets the current value of the ignoring the own IP addresses"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
 ["get <JAIL> ignorecommand", "gets ignorecommand of <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 72bf47d8..b425060a 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -76,6 +76,8 @@ class Filter(JailThread):
 		self.setUseDns(useDns)
 		## The amount of time to look back.
 		self.__findTime = 600
+		## Ignore own IPs flag:
+		self.__ignoreSelf = True
 		## The ignore IP list.
 		self.__ignoreIpList = []
 		## Size of line buffer
@@ -413,6 +415,17 @@ class Filter(JailThread):
 
 		return ip
 
+	##
+	# Ignore own IP/DNS.
+	#
+	@property
+	def ignoreSelf(self):
+		return self.__ignoreSelf
+
+	@ignoreSelf.setter
+	def ignoreSelf(self, value):
+		self.__ignoreSelf = value
+
 	##
 	# Add an IP/DNS to the ignore list.
 	#
@@ -458,6 +471,11 @@ class Filter(JailThread):
 	def inIgnoreIPList(self, ip, log_ignore=False):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
+
+		# check own IPs should be ignored and 'ip' is self IP:
+		if self.__ignoreSelf and ip in DNSUtils.getSelfIPs():
+			return True
+
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index bd3b812d..bda32ae8 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -118,6 +118,42 @@ class DNSUtils:
 
 		return ipList
 
+	@staticmethod
+	def getSelfNames():
+		"""Get own host names of self"""
+		# try find cached own hostnames (this tuple-key cannot be used elsewhere):
+		key = ('self','dns')
+		names = DNSUtils.CACHE_ipToName.get(key)
+		# get it using different ways (a set with names of localhost, hostname, fully qualified):
+		if names is None:
+			names = set(['localhost'])
+			for hostname in (socket.gethostname, socket.getfqdn):
+				try:
+					names |= set([hostname()])
+				except Exception as e: # pragma: no cover
+					logSys.warning("Retrieving own hostnames failed: %s", e)
+		# cache and return :
+		DNSUtils.CACHE_ipToName.set(key, names)
+		return names
+
+	@staticmethod
+	def getSelfIPs():
+		"""Get own IP addresses of self"""
+		# try find cached own IPs (this tuple-key cannot be used elsewhere):
+		key = ('self','ips')
+		ips = DNSUtils.CACHE_nameToIp.get(key)
+		# get it using different ways (a set with IPs of localhost, hostname, fully qualified):
+		if ips is None:
+			ips = set()
+			for hostname in DNSUtils.getSelfNames():
+				try:
+					ips |= set(DNSUtils.textToIp(hostname, 'yes'))
+				except Exception as e: # pragma: no cover
+					logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e)
+		# cache and return :
+		DNSUtils.CACHE_nameToIp.set(key, ips)
+		return ips
+
 
 ##
 # Class for IP address handling.
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index dfab1e38..facbe393 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -308,6 +308,12 @@ class Server:
 		return self.__jails[name].idle
 	
 	# Filter
+	def setIgnoreSelf(self, name, value):
+		self.__jails[name].filter.ignoreSelf = value
+	
+	def getIgnoreSelf(self, name):
+		return self.__jails[name].filter.ignoreSelf
+
 	def addIgnoreIP(self, name, ip):
 		self.__jails[name].filter.addIgnoreIP(ip)
 	
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index ad21b851..bc9edd43 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -181,6 +181,10 @@ class Transmitter:
 				raise Exception("Invalid idle option, must be 'on' or 'off'")
 			return self.__server.getIdleJail(name)
 		# Filter
+		elif command[1] == "ignoreself":
+			value = command[2]
+			self.__server.setIgnoreSelf(name, value)
+			return self.__server.getIgnoreSelf(name)
 		elif command[1] == "addignoreip":
 			value = command[2]
 			self.__server.addIgnoreIP(name, value)
@@ -341,6 +345,8 @@ class Transmitter:
 			return self.__server.getLogEncoding(name)
 		elif command[1] == "journalmatch": # pragma: systemd no cover
 			return self.__server.getJournalMatch(name)
+		elif command[1] == "ignoreself":
+			return self.__server.getIgnoreSelf(name)
 		elif command[1] == "ignoreip":
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "ignorecommand":
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index d3217555..10310a5d 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -325,6 +325,17 @@ class IgnoreIP(LogCaptureTestCase):
 		LogCaptureTestCase.setUp(self)
 		self.jail = DummyJail()
 		self.filter = FileFilter(self.jail)
+		self.filter.ignoreSelf = False
+
+	def testIgnoreSelfIP(self):
+		ipList = ("127.0.0.1",)
+		# test ignoreSelf is false:
+		for ip in ipList:
+			self.assertFalse(self.filter.inIgnoreIPList(ip))
+		# test ignoreSelf with true:
+		self.filter.ignoreSelf = True
+		for ip in ipList:
+			self.assertTrue(self.filter.inIgnoreIPList(ip))
 
 	def testIgnoreIPOK(self):
 		ipList = "127.0.0.1", "192.168.0.1", "255.255.255.255", "99.99.99.99"
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 8d1cfa62..51ff8880 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -449,6 +449,16 @@ class Transmitter(TransmitterBase):
 			self.transm.proceed(["set", self.jailName, "delignoreip", value]),
 			(0, [value]))
 
+		self.assertEqual(
+			self.transm.proceed(["get", self.jailName, "ignoreself"]),
+			(0, True))
+		self.assertEqual(
+			self.transm.proceed(["set", self.jailName, "ignoreself", False]),
+			(0, False))
+		self.assertEqual(
+			self.transm.proceed(["get", self.jailName, "ignoreself"]),
+			(0, False))
+
 	def testJailIgnoreCommand(self):
 		self.setGetTest("ignorecommand", "bin ", jail=self.jailName)
 

From 6c4b1c720475613b4765d040b4d86857a4ccf52b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Mar 2017 15:54:53 +0100
Subject: [PATCH 350/623] Update ChangeLog

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7629fb2f..b6e7bacd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -68,6 +68,10 @@ TODO: implementing of options resp. other tasks from PR #1346
 * Samples test case factory extended with filter options - dict in JSON to control 
   filter options (e. g. mode, etc.):
   # filterOptions: {"mode": "aggressive"}
+* Introduced new jail option "ignoreself", specifies whether the local resp. own IP addresses 
+  should be ignored (default is true). Fail2ban will not ban a host which matches such addresses.
+  Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS 
+  resp. IPs of the host self.
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From 663bc9903d69e95e19b67a0bf77c462a4f5fb5ad Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Mar 2017 16:19:21 +0100
Subject: [PATCH 351/623] increase coverage (was decreased since "ignoreip" was
 set to default empty)

---
 fail2ban/tests/fail2banclienttestcase.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 683368ee..35e00421 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -780,6 +780,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"findtime = 10m",
 				"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>",
 				"datepattern = {^LN-BEG}EPOCH",
+				"ignoreip = 127.0.0.1/8 ::1", # just to cover ignoreip in jailreader/transmitter
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",

From 30352c5f03263600e809fb3fa857cd99df1ba40d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Mar 2017 17:48:52 +0100
Subject: [PATCH 352/623] fix sporadic coverage changes (sometimes produces "no
 such process" in popen.poll after terminate/kill in timeout test cases)

---
 fail2ban/server/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py
index 58363e76..58363ff0 100644
--- a/fail2ban/server/utils.py
+++ b/fail2ban/server/utils.py
@@ -211,7 +211,8 @@ class Utils():
 				if retcode is None or tout_kill_tree: # Still going...
 					os.killpg(pgid, signal.SIGKILL) # Kill the process
 					time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
-					retcode = popen.poll()
+					if retcode is None: # pragma: no cover - too sporadic
+						retcode = popen.poll()
 					#logSys.debug("%s -- killed %s ", realCmd, retcode)
 				if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover
 					retcode = signal.SIGKILL

From e7052e9625926370365de5cef02e08e8ac548e3b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Mar 2017 09:55:20 +0100
Subject: [PATCH 353/623] update man/jail.conf.5 (docu for the ignoreself)

---
 man/jail.conf.5 | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 2e333e5a..5a75369c 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -199,11 +199,14 @@ Arguments can be passed to actions to override the default values from the [Init
 Values can also be quoted (required when value includes a ","). More that one action can be specified (in separate lines).
 .RE
 .TP
+.B ignoreself
+boolean value (default true) indicates the banning of own IP addresses should be prevented
+.TP
 .B ignoreip
-list of IPs not to ban. They can include a CIDR mask too.
+list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host.
 .TP
 .B ignorecommand
-command that is executed to determine if the current candidate IP for banning should not be banned.
+command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit.
 .br
 IP will not be banned if command returns successfully (exit code 0).
 Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution. Currently only <ip> is supported however more will be added later.

From 61c1bdfe79a98be3f1aa7ccb2957bf2299d768fc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Mar 2017 22:02:37 +0100
Subject: [PATCH 354/623] Normalizes replacement of `<SKIPLINES>` (moved to
 _resolveHostTag, so will be replaced together with another tags); Regex will
 be compiled as MULTILINE only if needed (buffering with `maxlines` > 1), that
 enables: - improve performance by the single line parsing; - make regex more
 precise (because distinguish between anchors `^`/`$` for the begin/end of
 string and the new-line character '\n', e. g. if coming from filters (like
 systemd journal) that allow the parsing of log-entries contain new-line chars
 (as single entry);

---
 fail2ban/server/failregex.py      | 26 ++++++++++++++++++--------
 fail2ban/server/filter.py         |  8 ++++++--
 fail2ban/tests/samplestestcase.py |  2 +-
 3 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index 59f59978..c1e7107a 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -103,20 +103,17 @@ class Regex:
 	# avoid construction of invalid object.
 	# @param value the regular expression
 	
-	def __init__(self, regex, **kwargs):
+	def __init__(self, regex, multiline=False, **kwargs):
 		self._matchCache = None
 		# Perform shortcuts expansions.
-		# Resolve "<HOST>" tag using default regular expression for host:
+		# Replace standard f2b-tags (like "<HOST>", etc) using default regular expressions:
 		regex = Regex._resolveHostTag(regex, **kwargs)
-		# Replace "<SKIPLINES>" with regular expression for multiple lines.
-		regexSplit = regex.split("<SKIPLINES>")
-		regex = regexSplit[0]
-		for n, regexLine in enumerate(regexSplit[1:]):
-			regex += "\n(?P<skiplines%i>(?:(.*\n)*?))" % n + regexLine
+		#
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
+		flags = re.MULTILINE if (multiline or "\n" in regex or r"\n" in regex) else 0
 		try:
-			self._regexObj = re.compile(regex, re.MULTILINE)
+			self._regexObj = re.compile(regex, flags)
 			self._regex = regex
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
@@ -125,6 +122,11 @@ class Regex:
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
 
+	@property
+	def flags(self):
+		"""Returns the regex matching flags combination of the compiled regex object"""
+		return self._regexObj.flags
+
 	##
 	# Replaces "<HOST>", "<IP4>", "<IP6>", "<FID>" with default regular expression for host
 	#
@@ -135,6 +137,9 @@ class Regex:
 	def _resolveHostTag(regex, useDns="yes"):
 
 		openTags = dict()
+		props = {
+			'nl': 0, # new lines counter by <SKIPLINES> tag;
+		}
 		# tag interpolation callable:
 		def substTag(m):
 			tag = m.group()
@@ -142,6 +147,11 @@ class Regex:
 			# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
 			if tn == "HOST":
 				return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR]
+			# replace "<SKIPLINES>" with regular expression for multiple lines (by buffering with maxlines)
+			if tn == "SKIPLINES":
+				nl = props['nl']
+				props['nl'] = nl + 1
+				return r"\n(?P<skiplines%i>(?:(?:.*\n)*?))" % (nl,)
 			# static replacement from RH4TAG:
 			try:
 				return RH4TAG[tn]
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b425060a..ed50e8c1 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -161,10 +161,14 @@ class Filter(JailThread):
 	# @param value the regular expression
 
 	def addFailRegex(self, value):
+		multiLine = self.getMaxLines() > 1
 		try:
-			regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns)
+			regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine,
+				useDns=self.__useDns)
 			self.__failRegex.append(regex)
-			if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
+			regexExpr = regex.getRegex()
+			# check new lines present in regex (was compiled as multiline), incorrect by `maxlines=1`:
+			if (regex.flags & re.MULTILINE) and not multiLine:
 				logSys.warning(
 					"Mutliline regex set for jail %r "
 					"but maxlines not greater than 1", self.jailName)
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 0ba11c2e..fb6812cb 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -41,7 +41,7 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 # regexp to test greedy catch-all should be not-greedy:
 RE_HOST = Regex('<HOST>').getRegex()
-RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?).*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
+RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
 
 
 class FilterSamplesRegex(unittest.TestCase):

From bc888e07533bcebd81a9fedf7ce846d86851d63b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Mar 2017 12:05:51 +0100
Subject: [PATCH 355/623] Regex compiled in multi-line parsing mode only if
 `maxlines` > 1 (buffering), if however expected - prefix `(?m)` could be used
 in regex to enable it; Removed warning "Mutliline regex set for jail ... but
 maxlines not greater than 1", because can be expected situation now: non
 multi-line entry from systemd-filter containing new-lines (that should be
 ignored by anchors resp. entry parsed as single string); small code review;

---
 fail2ban/server/failregex.py      | 8 +-------
 fail2ban/server/filter.py         | 6 ------
 fail2ban/tests/filtertestcase.py  | 6 +++---
 fail2ban/tests/samplestestcase.py | 2 +-
 4 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py
index c1e7107a..d5c9345f 100644
--- a/fail2ban/server/failregex.py
+++ b/fail2ban/server/failregex.py
@@ -111,9 +111,8 @@ class Regex:
 		#
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
-		flags = re.MULTILINE if (multiline or "\n" in regex or r"\n" in regex) else 0
 		try:
-			self._regexObj = re.compile(regex, flags)
+			self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
 			self._regex = regex
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
@@ -122,11 +121,6 @@ class Regex:
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
 
-	@property
-	def flags(self):
-		"""Returns the regex matching flags combination of the compiled regex object"""
-		return self._regexObj.flags
-
 	##
 	# Replaces "<HOST>", "<IP4>", "<IP6>", "<FID>" with default regular expression for host
 	#
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index ed50e8c1..75536d57 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -166,12 +166,6 @@ class Filter(JailThread):
 			regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine,
 				useDns=self.__useDns)
 			self.__failRegex.append(regex)
-			regexExpr = regex.getRegex()
-			# check new lines present in regex (was compiled as multiline), incorrect by `maxlines=1`:
-			if (regex.flags & re.MULTILINE) and not multiLine:
-				logSys.warning(
-					"Mutliline regex set for jail %r "
-					"but maxlines not greater than 1", self.jailName)
 		except RegexException as e:
 			logSys.error(e)
 			raise e
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 10310a5d..ce665e72 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1479,8 +1479,8 @@ class GetFailures(LogCaptureTestCase):
 		output = [("192.0.43.10", 2, 1124013599.0),
 			("192.0.43.11", 1, 1124013598.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
-		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxLines(100)
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
@@ -1497,9 +1497,9 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
+		self.filter.setMaxLines(100)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addIgnoreRegex("rsync error: Received SIGINT")
-		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
@@ -1513,9 +1513,9 @@ class GetFailures(LogCaptureTestCase):
 			("192.0.43.11", 1, 1124013598.0),
 			("192.0.43.15", 1, 1124013598.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
+		self.filter.setMaxLines(100)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")
-		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index fb6812cb..121c1c5c 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -40,7 +40,7 @@ TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 # regexp to test greedy catch-all should be not-greedy:
-RE_HOST = Regex('<HOST>').getRegex()
+RE_HOST = Regex._resolveHostTag('<HOST>')
 RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
 
 

From 990d9a66da60ceca5c1acca83ba425db8c78b726 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Mar 2017 17:07:21 +0100
Subject: [PATCH 356/623] fail2ban-regex: fixed matched output by multi-line
 (buffered) parsing + and multi-line debuggex URL; test coverage extended;

---
 fail2ban/client/fail2banregex.py        | 47 +++++++++++++++++++------
 fail2ban/tests/fail2banregextestcase.py | 37 +++++++++++++++++++
 2 files changed, 73 insertions(+), 11 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 782513a7..45dbfe95 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -55,11 +55,14 @@ from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack,
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
-def debuggexURL(sample, regex, useDns="yes"):
-	q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns),
-							'str': sample,
-							'flavor': 'python' })
-	return 'https://www.debuggex.com/?' + q
+def debuggexURL(sample, regex, multiline=False, useDns="yes"):
+	args = {
+		're': Regex._resolveHostTag(regex, useDns=useDns),
+		'str': sample,
+		'flavor': 'python'
+	}
+	if multiline: args['flags'] = 'm'
+	return 'https://www.debuggex.com/?' + urllib.urlencode(args)
 
 def output(args): # pragma: no cover (overriden in test-cases)
 	print(args)
@@ -400,6 +403,7 @@ class Fail2banRegex(object):
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
 			ret = self._filter.processLine(line, date)
+			lines = []
 			line = self._filter.processedLine()
 			for match in ret:
 				# Append True/False flag depending if line was matched by
@@ -422,9 +426,17 @@ class Fail2banRegex(object):
 								"".join(bufLine[::2])))
 				except ValueError:
 					pass
-				else:
-					self._line_stats.matched += 1
-					self._line_stats.missed -= 1
+				# if buffering - add also another lines from match:
+				if self._print_all_matched:
+					if not self._debuggex:
+						self._line_stats.matched_lines.append("".join(bufLine))
+					else:
+						lines.append(bufLine[0] + bufLine[2])
+				self._line_stats.matched += 1
+				self._line_stats.missed -= 1
+		if lines: # pre-lines parsed in multiline mode (buffering)
+			lines.append(line)
+			line = "\n".join(lines)
 		return line, ret
 
 	def process(self, test_lines):
@@ -472,6 +484,7 @@ class Fail2banRegex(object):
 		assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
 		lines = lstats[ltype]
 		l = lstats[ltype + '_lines']
+		multiline = self._filter.getMaxLines() > 1
 		if lines:
 			header = "%s line(s):" % (ltype.capitalize(),)
 			if self._debuggex:
@@ -485,7 +498,8 @@ class Fail2banRegex(object):
 					for arg in [l, regexlist]:
 						ans = [ x + [y] for x in ans for y in arg ]
 					b = map(lambda a: a[0] +  ' | ' + a[1].getFailRegex() + ' |  ' + 
-						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans)
+						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), 
+							multiline, self._opts.usedns), ans)
 					pprint_list([x.rstrip() for x in b], header)
 				else:
 					output( "%s too many to print.  Use --print-all-%s " \
@@ -599,8 +613,19 @@ class Fail2banRegex(object):
 			output( "Use    journal match : %s" % " ".join(journalmatch) )
 			test_lines = journal_lines_gen(flt, myjournal)
 		else:
-			output( "Use      single line : %s" % shortstr(cmd_log) )
-			test_lines = [ cmd_log ]
+			# if single line parsing (without buffering)
+			if self._filter.getMaxLines() <= 1:
+				output( "Use      single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
+				test_lines = [ cmd_log ]
+			else: # multi line parsing (with buffering)
+				test_lines = cmd_log.split("\n")
+				output( "Use      multi line : %s line(s)" % len(test_lines) )
+				for i, l in enumerate(test_lines):
+					if i >= 5:
+						output( "| ..." ); break
+					output( "| %2.2s: %s" % (i+1, shortstr(l)) )
+				output( "`-" )
+			
 		output( "" )
 
 		self.process(test_lines)
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 0cd0e303..d865b34d 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -252,6 +252,43 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		)
 		self.assertTrue(fail2banRegex.start(args))
 
+	def testDirectMultilineBuf(self):
+		# test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed):
+		for preLines in (0, 20):
+			self.pruneLog("[test-phase %s]" % preLines)
+			(opts, args, fail2banRegex) = _Fail2banRegex(
+				"--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5", 
+				("1490349000 TEST-NL\n"*preLines) + 
+				"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
+				r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
+			)
+			self.assertTrue(fail2banRegex.start(args))
+			self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2))
+			# both matched lines were printed:
+			self.assertLogged("|  1490349000 FAIL", "|  1490349001 HOST 192.0.2.34", all=True)
+
+
+	def testDirectMultilineBufDebuggex(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5",
+			"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
+			r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
+		self.assertLogged("&flags=m")
+
+	def testSinglelineWithNLinContent(self):
+		# 
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--usedns", "no", "-d", "^Epoch", "--print-all-matched",
+			"1490349000 FAIL: failure\nhost: 192.0.2.35",
+			r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$"
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
+
+
 	def testWrongFilterFile(self):
 		# use test log as filter file to cover eror cases...
 		(opts, args, fail2banRegex) = _Fail2banRegex(

From 6ac5c55edcf7548cb52beffab096d717cb5e46bd Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Mar 2017 17:35:41 +0100
Subject: [PATCH 357/623] the sequence in args-dict is currently undefined (so
 can be 1st argument with `?` instead of `&`)

---
 fail2ban/tests/fail2banregextestcase.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index d865b34d..8bfedad1 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -276,7 +276,8 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		)
 		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
-		self.assertLogged("&flags=m")
+		# the sequence in args-dict is currently undefined (so can be 1st argument)
+		self.assertLogged("&flags=m", "?flags=m")
 
 	def testSinglelineWithNLinContent(self):
 		# 

From c82495353fdc04b7b7a9ee8d63ef117ba4a1a9c9 Mon Sep 17 00:00:00 2001
From: Seth Reeser <seth.reeser@devopsgroup.io>
Date: Fri, 24 Mar 2017 14:03:20 -0400
Subject: [PATCH 358/623] Update mysqld-auth.conf (#1725)

---
 config/filter.d/mysqld-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf
index 3ad70cb7..31bd2056 100644
--- a/config/filter.d/mysqld-auth.conf
+++ b/config/filter.d/mysqld-auth.conf
@@ -1,4 +1,4 @@
-# Fail2Ban filter for unsuccesfull MySQL authentication attempts
+# Fail2Ban filter for unsuccesful MySQL authentication attempts
 #
 #
 # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]:

From d26060ead0a2b241c495bc8d777ab3e2cc9e788e Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 27 Mar 2017 09:38:53 +0200
Subject: [PATCH 359/623] Update ChangeLog

belongs to #1733
---
 ChangeLog | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b6e7bacd..eafaef0f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -27,6 +27,8 @@ TODO: implementing of options resp. other tasks from PR #1346
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`
   - fixed using new tag `<ip-host>` (without external command execution)
+* fail2ban-regex: fixed matched output by multi-line (buffered) parsing
+* fail2ban-regex: support for multi-line debuggex URL implemented (gh-422)
 
 ### New Features
 * New Actions:
@@ -72,6 +74,13 @@ TODO: implementing of options resp. other tasks from PR #1346
   should be ignored (default is true). Fail2ban will not ban a host which matches such addresses.
   Option "ignoreip" affects additionally to "ignoreself" and don't need to include the DNS 
   resp. IPs of the host self.
+* Regex will be compiled as MULTILINE only if needed (buffering with `maxlines` > 1), that enables:
+  - to improve performance by the single line parsing (see gh-1733);
+  - make regex more precise (because distinguish between anchors `^`/`$` for the begin/end of string
+    and the new-line character '\n', e. g. if coming from filters (like systemd journal) that allow
+    the parsing of log-entries contain new-line chars (as single entry);
+  - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` 
+    could be used in regex to enable it;
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From 7b93f111e1cd137a4d4935e276ae32b712d37008 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Tue, 28 Mar 2017 19:29:45 +0200
Subject: [PATCH 360/623] test_smtp inconsistency for py3+IPv6

It appears that, under Python3, on an IPv6 enabled machine,
the testing SMTP server on 'localhost' can turn out to listen on ::1 only,
which makes those tests break if the SMTP client part uses 127.0.0.1
directly. Using 'localhost' there as well makes the tests pass.
---
 fail2ban/tests/action_d/test_smtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index d262f73e..d0858b85 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -67,7 +67,7 @@ class SMTPActionTest(unittest.TestCase):
 		port = self.smtpd.socket.getsockname()[1]
 
 		self.action = customActionModule.Action(
-			self.jail, "test", host="127.0.0.1:%i" % port)
+			self.jail, "test", host="localhost:%i" % port)
 
 		## because of bug in loop (see loop in asyncserver.py) use it's loop instead of asyncore.loop:
 		self._active = True

From 7437fbd75be2df301bdc9acfcf8e8d1381cbacda Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Mar 2017 18:54:28 +0200
Subject: [PATCH 361/623] strptime.py: small code review and performance
 optimization (get some properties on demand, etc.)

---
 fail2ban/server/strptime.py | 77 +++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 41 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index cdfe0e0e..d98b2e1b 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -95,18 +95,10 @@ def reGroupDictStrptime(found_dict, msec=False):
 		Unix time stamp.
 	"""
 
-	now = MyTime.now()
-	year = month = day = hour = minute = None
-	hour = minute = None
+	now = \
+	year = month = day = hour = minute = tzoffset = \
+	weekday = julian = week_of_year = None
 	second = fraction = 0
-	tzoffset = None
-	# Default to -1 to signify that values not known; not critical to have,
-	# though
-	week_of_year = -1
-	week_of_year_start = -1
-	# weekday and julian defaulted to -1 so as to signal need to calculate
-	# values
-	weekday = julian = -1
 	for key, val in found_dict.iteritems():
 		if val is None: continue
 		# Directives not explicitly handled below:
@@ -199,31 +191,28 @@ def reGroupDictStrptime(found_dict, msec=False):
 	# Fail2Ban will assume it's this year
 	assume_year = False
 	if year is None:
+		if not now: now = MyTime.now()
 		year = now.year
 		assume_year = True
-	# If we know the week of the year and what day of that week, we can figure
-	# out the Julian day of the year.
-	if julian == -1 and week_of_year != -1 and weekday != -1:
-		week_starts_Mon = True if week_of_year_start == 0 else False
-		julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
-											week_starts_Mon)
-	# Cannot pre-calculate datetime.datetime() since can change in Julian
-	# calculation and thus could have different value for the day of the week
-	# calculation.
-	if julian != -1 and (month is None or day is None):
-		datetime_result = datetime.datetime.fromordinal((julian - 1) + datetime.datetime(year, 1, 1).toordinal())
-		year = datetime_result.year
-		month = datetime_result.month
-		day = datetime_result.day
-	# Add timezone info
-	if tzoffset is not None:
-		gmtoff = tzoffset * 60
-	else:
-		gmtoff = None
+	if month is None or day is None:
+		# If we know the week of the year and what day of that week, we can figure
+		# out the Julian day of the year.
+		if julian is None and week_of_year is not None and weekday is not None:
+			julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+												(week_of_year_start == 0))
+		# Cannot pre-calculate datetime.datetime() since can change in Julian
+		# calculation and thus could have different value for the day of the week
+		# calculation.
+		if julian is not None:
+			datetime_result = datetime.datetime.fromordinal((julian - 1) + datetime.datetime(year, 1, 1).toordinal())
+			year = datetime_result.year
+			month = datetime_result.month
+			day = datetime_result.day
 
 	# Fail2Ban assume today
 	assume_today = False
 	if month is None and day is None:
+		if not now: now = MyTime.now()
 		month = now.month
 		day = now.day
 		assume_today = True
@@ -231,19 +220,25 @@ def reGroupDictStrptime(found_dict, msec=False):
 	# Actully create date
 	date_result =  datetime.datetime(
 		year, month, day, hour, minute, second, fraction)
-	if gmtoff is not None:
-		date_result = date_result - datetime.timedelta(seconds=gmtoff)
+	# Add timezone info
+	if tzoffset is not None:
+		date_result -= datetime.timedelta(seconds=tzoffset * 60)
 
-	if date_result > now and assume_today:
-		# Rollover at midnight, could mean it's yesterday...
-		date_result = date_result - datetime.timedelta(days=1)
-	if date_result > now and assume_year:
-		# Could be last year?
-		# also reset month and day as it's not yesterday...
-		date_result = date_result.replace(
-			year=year-1, month=month, day=day)
+	if assume_today:
+		if not now: now = MyTime.now()
+		if date_result > now:
+			# Rollover at midnight, could mean it's yesterday...
+			date_result -= datetime.timedelta(days=1)
+	if assume_year:
+		if not now: now = MyTime.now()
+		if date_result > now:
+			# Could be last year?
+			# also reset month and day as it's not yesterday...
+			date_result = date_result.replace(
+				year=year-1, month=month, day=day)
 
-	if gmtoff is not None:
+	# make time:
+	if tzoffset is not None:
 		tm = calendar.timegm(date_result.utctimetuple())
 	else:
 		tm = time.mktime(date_result.timetuple())

From ee3c9fcb75091b1d28cfcaf2535232ffe3dc465a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Mar 2017 22:07:40 +0200
Subject: [PATCH 362/623] "%y" - in the fail2ban parsed year without century
 should be always relative current century (>= 2000); cover several format
 specifiers and different "assume" cases (without year, without date, greater
 as now, etc.);

---
 fail2ban/server/strptime.py            | 27 +++++------------
 fail2ban/tests/datedetectortestcase.py | 41 ++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index d98b2e1b..55bdcc8c 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -108,13 +108,9 @@ def reGroupDictStrptime(found_dict, msec=False):
 		#	  worthless without day of the week
 		if key == 'y':
 			year = int(val)
-			# Open Group specification for strptime() states that a %y
-			#value in the range of [00, 68] is in the century 2000, while
-			#[69,99] is in the century 1900
-			if year <= 68:
+			# Fail2ban year should be always in the current century (>= 2000)
+			if year <= 2000:
 				year += 2000
-			else:
-				year += 1900
 		elif key == 'Y':
 			year = int(val)
 		elif key == 'm':
@@ -148,7 +144,7 @@ def reGroupDictStrptime(found_dict, msec=False):
 		elif key == 'S':
 			second = int(val)
 		elif key == 'f':
-			if msec:
+			if msec: # pragma: no cover - currently unused
 				s = val
 				# Pad to always return microseconds.
 				s += "0" * (6 - len(s))
@@ -158,21 +154,14 @@ def reGroupDictStrptime(found_dict, msec=False):
 		elif key == 'a':
 			weekday = locale_time.a_weekday.index(val.lower())
 		elif key == 'w':
-			weekday = int(val)
-			if weekday == 0:
-				weekday = 6
-			else:
-				weekday -= 1
+			weekday = int(val) - 1
+			if weekday < 0: weekday = 6
 		elif key == 'j':
 			julian = int(val)
 		elif key in ('U', 'W'):
 			week_of_year = int(val)
-			if key == 'U':
-				# U starts week on Sunday.
-				week_of_year_start = 6
-			else:
-				# W starts week on Monday.
-				week_of_year_start = 0
+			# U starts week on Sunday, W - on Monday
+			week_of_year_start = 6 if key == 'U' else 0
 		elif key == 'z':
 			z = val
 			if z in ("Z", "UTC", "GMT"):
@@ -242,6 +231,6 @@ def reGroupDictStrptime(found_dict, msec=False):
 		tm = calendar.timegm(date_result.utctimetuple())
 	else:
 		tm = time.mktime(date_result.timetuple())
-	if msec:
+	if msec: # pragma: no cover - currently unused
 		tm += fraction/1000000.0
 	return tm
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 5b32a7e9..39ab7173 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -298,6 +298,16 @@ iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
 
 class CustomDateFormatsTest(unittest.TestCase):
 
+	def setUp(self):
+		"""Call before every test case."""
+		unittest.TestCase.setUp(self)
+		setUpMyTime()
+
+	def tearDown(self):
+		"""Call after every test case."""
+		unittest.TestCase.tearDown(self)
+		tearDownMyTime()
+
 	def testIso8601(self):
 		date = datetime.datetime.utcfromtimestamp(
 			iso8601.getDate("2007-01-25T12:00:00Z")[0])
@@ -411,6 +421,37 @@ class CustomDateFormatsTest(unittest.TestCase):
 			else:
 				self.assertEqual(date, None)
 
+	def testVariousFormatSpecs(self):
+		for (matched, dp, line) in (
+			# cover %B (full-month-name) and %I (as 12 == 0):
+			(1106438399.0, "^%B %Exd %I:%ExM:%ExS**", 'January 23 12:59:59'),
+			# cover %U (week of year starts on sunday) and %A (weekday):
+			(985208399.0,  "^%y %U %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'),
+			# cover %W (week of year starts on monday) and %A (weekday):
+			(984603599.0,  "^%y %W %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'),
+			# cover %W (week of year starts on monday) and %w (weekday, 0 - sunday):
+			(984949199.0,  "^%y %W %w %ExH:%ExM:%ExS**", '01 11 0 21:59:59'),
+			# cover %W (week of year starts on monday) and %w (weekday, 6 - saturday):
+			(984862799.0,  "^%y %W %w %ExH:%ExM:%ExS**", '01 11 6 21:59:59'),
+			# cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> back to yesterday (13 Aug):
+			(1123963199.0,  "^%ExH:%ExM:%ExS**", '21:59:59'),
+			# cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> today (14 Aug):
+			(1123970401.0,  "^%ExH:%ExM:%ExS**", '00:00:01'),
+			# cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004):
+			(1094068799.0,  "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'),
+		):
+			logSys.debug('== test: %r', (matched, dp, line))
+			dd = DateDetector()
+			dd.appendTemplate(dp)
+			date = dd.getTime(line)
+			if matched:
+				self.assertTrue(date)
+				if isinstance(matched, basestring): # pragma: no cover
+					self.assertEqual(matched, date[1].group(1))
+				else:
+					self.assertEqual(matched, date[0])
+			else: # pragma: no cover
+				self.assertEqual(date, None)
 
 #	def testDefaultTempate(self):
 #		self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")

From 05f5c6efccc22a728cd0d21659747f9335fa1daf Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 29 Mar 2017 12:32:34 +0200
Subject: [PATCH 363/623] Update README.md

added wiki-reference;
fixed mail-representation (after github swiched markdown syntax)
---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ee654acb..72c48378 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,8 @@ mechanisms if you really want to protect services.
 ------|------
 
 This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs
-are available in fail2ban(1) manpage and on the website http://www.fail2ban.org
+are available in fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
+and on the website http://www.fail2ban.org
 
 Installation:
 -------------
@@ -89,7 +90,7 @@ Contact:
 See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md)
 
 ### You just appreciate this program:
-send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier <cyril.jaquier@fail2ban.org>))
+send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org))
 or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users)
 since Fail2Ban is "community-driven" for years now.
 

From 8bf79fa4837a8c8f4d92e107fdf8585af41b434e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 17:44:15 +0200
Subject: [PATCH 364/623] implemented execution of `actionstart` on demand, if
 action depends on `family` (closes gh-1741); new action parameter
 "actionstart_on_demand" (bool) can be set to prevent/allow starting action on
 demand (default retrieved automatically, if some conditional parameter
 `param?family=...` presents in action properties);

---
 config/action.d/pf.conf          |  5 ++-
 fail2ban/client/actionreader.py  |  7 ++-
 fail2ban/server/action.py        | 73 ++++++++++++++++++++++++++-----
 fail2ban/tests/servertestcase.py | 74 ++++++++++++++++++++++++--------
 4 files changed, 125 insertions(+), 34 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index b7476fa2..deb38c09 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -18,6 +18,9 @@
 actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
               echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -f-
 
+# Option:  start_on_demand - to start action on demand
+# Example: `action=pf[actionstart_on_demand=true]`
+actionstart_on_demand = false
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -71,8 +74,6 @@ tablename = f2b
 #
 protocol = tcp
 
-
-
 # Option: actiontype
 # Notes.: defines additions to the blocking rule
 # Values: leave empty to block all attempts from the host
diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index 0fd55f41..b85f22a0 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -38,6 +38,7 @@ class ActionReader(DefinitionInitConfigReader):
 
 	_configOpts = {
 		"actionstart": ["string", None],
+		"actionstart_on_demand": ["string", None],
 		"actionstop": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
@@ -73,8 +74,10 @@ class ActionReader(DefinitionInitConfigReader):
 		opts = self.getCombined(
 			ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
 		# type-convert only after combined (otherwise boolean converting prevents substitution):
-		if opts.get('norestored'):
-			opts['norestored'] = self._convert_to_boolean(opts['norestored'])
+		for o in ('norestored', 'actionstart_on_demand'):
+			if opts.get(o):
+				opts[o] = self._convert_to_boolean(opts[o])
+		
 		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 1c5eb0c9..8afeea6e 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -50,6 +50,8 @@ allowed_ipv6 = True
 # capture groups from filter for map to ticket data:
 FCUSTAG_CRE = re.compile(r'<F-([A-Z0-9_\-]+)>'); # currently uppercase only
 
+CONDITIONAL_FAM_RE = re.compile(r"^(\w+)\?(family)=")
+
 # New line, space
 ADD_REPL_TAGS = {
   "br": "\n", 
@@ -290,6 +292,7 @@ class CommandAction(ActionBase):
 		super(CommandAction, self).__init__(jail, name)
 		self.__init = 1
 		self.__properties = None
+		self.__started = {}
 		self.__substCache = {}
 		self.clearAllParams()
 		self._logSys.debug("Created %s" % self.__class__)
@@ -342,7 +345,11 @@ class CommandAction(ActionBase):
 	def _substCache(self):
 		return self.__substCache
 
-	def _executeOperation(self, tag, operation):
+	def _getOperation(self, tag, family):
+		return self.replaceTag(tag, self._properties,
+			conditional=('family=' + family), cache=self.__substCache)
+
+	def _executeOperation(self, tag, operation, family=[]):
 		"""Executes the operation commands (like "actionstart", "actionstop", etc).
 
 		Replace the tags in the action command with actions properties
@@ -352,14 +359,14 @@ class CommandAction(ActionBase):
 		res = True
 		try:
 			# common (resp. ipv4):
-			startCmd = self.replaceTag(tag, self._properties, 
-				conditional='family=inet4', cache=self.__substCache)
-			if startCmd:
-				res &= self.executeCmd(startCmd, self.timeout)
+			startCmd = None
+			if not family or 'inet4' in family:
+				startCmd = self._getOperation(tag, 'inet4')
+				if startCmd:
+					res &= self.executeCmd(startCmd, self.timeout)
 			# start ipv6 actions if available:
-			if allowed_ipv6:
-				startCmd6 = self.replaceTag(tag, self._properties, 
-					conditional='family=inet6', cache=self.__substCache)
+			if allowed_ipv6 and (not family or 'inet6' in family):
+				startCmd6 = self._getOperation(tag, 'inet6')
 				if startCmd6 and startCmd6 != startCmd:
 					res &= self.executeCmd(startCmd6, self.timeout)
 			if not res:
@@ -367,13 +374,34 @@ class CommandAction(ActionBase):
 		except ValueError as e:
 			raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e))
 
-	def start(self):
+	COND_FAMILIES = {'inet4':1, 'inet6':1}
+
+	@property
+	def _startOnDemand(self):
+		"""Checks the action depends on family (conditional)"""
+		v = self._properties.get('actionstart_on_demand')
+		if v is None:
+			v = False
+			for n in self._properties:
+				if CONDITIONAL_FAM_RE.match(n):
+					v = True
+					break
+		self._properties['actionstart_on_demand'] = v
+		return v
+
+	def start(self, family=[]):
 		"""Executes the "actionstart" command.
 
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		return self._executeOperation('<actionstart>', 'starting')
+		if not family:
+			# check the action depends on family (conditional):
+			if self._startOnDemand:
+				return True
+		elif self.__started.get(family):
+			return True
+		return self._executeOperation('<actionstart>', 'starting', family=family)
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -387,6 +415,20 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
+		# if we should start the action on demand (conditional by family):
+		if self._startOnDemand:
+			family = aInfo.get('family')
+			if not self.__started.get(family):
+				self.start(family)
+				self.__started[family] = 1
+				# mark also another families as "started" (-1), if they are equal
+				# (on demand, but the same for ipv4 and ipv6):
+				cmd = self._getOperation('<actionstart>', family)
+				for f in CommandAction.COND_FAMILIES:
+					if f != family and not self.__started.get(f):
+						if cmd == self._getOperation('<actionstart>', f):
+							self.__started[f] = -1
+		# ban:
 		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
@@ -411,7 +453,16 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		return self._executeOperation('<actionstop>', 'stopping')
+		family = []
+		# cumulate started families, if started on demand (conditional):
+		if self._startOnDemand:
+			for f in CommandAction.COND_FAMILIES:
+				if self.__started.get(f) == 1: # only real started:
+					family.append(f)
+					self.__started[f] = 0
+			# if no started (on demand) actions:
+			if not family: return True
+		return self._executeOperation('<actionstop>', 'stopping', family=family)
 
 	def reload(self, **kwargs):
 		"""Executes the "actionreload" command.
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 51ff8880..603bb69f 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1185,10 +1185,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-multiport --
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
 						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
 						"`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+					), 
+					'ip6-start': (
 						"`ip6tables -w -N f2b-j-w-iptables-mp`",
 						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
@@ -1223,10 +1225,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-allports --
 				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
 						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
 						"`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+					), 
+					'ip6-start': (
 						"`ip6tables -w -N f2b-j-w-iptables-ap`",
 						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
@@ -1261,9 +1265,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-ipset-proto6 --
 				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
-					'start': (
+					'ip4-start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
 						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+					), 
+					'ip6-start': (
 						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
@@ -1293,9 +1299,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-ipset-proto6-allports --
 				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
-					'start': (
+					'ip4-start': (
 						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
 						"`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+					), 
+					'ip6-start': (
 						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
@@ -1325,10 +1333,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables --
 				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables`",
 						"`iptables -w -A f2b-j-w-iptables -j RETURN`",
 						"`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+					), 
+					'ip6-start': (
 						"`ip6tables -w -N f2b-j-w-iptables`",
 						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
@@ -1363,10 +1373,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-new --
 				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-new`",
 						"`iptables -w -A f2b-j-w-iptables-new -j RETURN`",
 						"`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+					), 
+					'ip6-start': (
 						"`ip6tables -w -N f2b-j-w-iptables-new`",
 						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
 						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
@@ -1401,8 +1413,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# iptables-xt_recent-echo --
 				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
 					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
-					'start': (
+					'ip4-start': (
 						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+					), 
+					'ip6-start': (
 						"`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
 					),
 					'stop': (
@@ -1431,7 +1445,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),
 				}),
 				# pf default -- multiport on default port (tag <port> set in jail.conf, but not in this test case)
-				('j-w-pf', 'pf[name=%(__name__)s]', {
+				('j-w-pf', 'pf[name=%(__name__)s, actionstart_on_demand=false]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
@@ -1468,13 +1482,14 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
-				# pf allports --
-				('j-w-pf-ap', 'pf[actiontype=<allports>][name=%(__name__)s]', {
+				# pf allports -- test additionally "actionstart_on_demand" was set to true
+				('j-w-pf-ap', 'pf[actiontype=<allports>, actionstart_on_demand=true][name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
-					'start': (
+					'ip4-start': (
 						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',
 						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -f-`',
 					),
+					'ip6-start': (), # the same as ipv4
 					'stop': (
 						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -f-`',
 						'`pfctl -t f2b-j-w-pf-ap -T flush`',
@@ -1490,10 +1505,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
 						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+					), 
+					'ip6-start': (
 						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
 						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
@@ -1528,10 +1545,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# firewallcmd-allports --
 				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
-					'start': (
+					'ip4-start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
 						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+					), 
+					'ip6-start': (
 						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
 						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
@@ -1566,9 +1585,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				# firewallcmd-ipset --
 				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
 					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
-					'start': (
+					'ip4-start': (
 						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",
 						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+					), 
+					'ip6-start': (
 						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`",
 						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
@@ -1614,6 +1635,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 			jails = server._Server__jails
 
+			tickets = {
+				'ip4': BanTicket('192.0.2.1'),
+				'ip6': BanTicket('2001:DB8::'),
+			}
 			for jail, act, tests in testJailsActions:
 				# print(jail, jails[jail])
 				for a in jails[jail].actions:
@@ -1627,25 +1652,36 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# test start :
 					self.pruneLog('# === start ===')
 					action.start()
-					self.assertLogged(*tests['start'], all=True)
+					if tests.get('start'):
+						self.assertLogged(*tests['start'], all=True)
+					else:
+						self.assertNotLogged(*tests['ip4-start']+tests['ip6-start'], all=True)
+					ainfo = {
+						'ip4': _actions.Actions.ActionInfo(tickets['ip4'], jails[jail]),
+						'ip6': _actions.Actions.ActionInfo(tickets['ip6'], jails[jail]),
+					}
 					# test ban ip4 :
 					self.pruneLog('# === ban-ipv4 ===')
-					action.ban({'ip': IPAddr('192.0.2.1')})
+					action.ban(ainfo['ip4'])
+					if tests.get('ip4-start'): self.assertLogged(*tests['ip4-start'], all=True)
+					if tests.get('ip6-start'): self.assertNotLogged(*tests['ip6-start'], all=True)
 					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test unban ip4 :
 					self.pruneLog('# === unban ipv4 ===')
-					action.unban({'ip': IPAddr('192.0.2.1')})
+					action.unban(ainfo['ip4'])
 					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test ban ip6 :
 					self.pruneLog('# === ban ipv6 ===')
-					action.ban({'ip': IPAddr('2001:DB8::')})
+					action.ban(ainfo['ip6'])
+					if tests.get('ip6-start'): self.assertLogged(*tests['ip6-start'], all=True)
+					if tests.get('ip4-start'): self.assertNotLogged(*tests['ip4-start'], all=True)
 					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test unban ip6 :
 					self.pruneLog('# === unban ipv6 ===')
-					action.unban({'ip': IPAddr('2001:DB8::')})
+					action.unban(ainfo['ip6'])
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test stop :

From ca18270beb046279d0bf6dfd011fcedad59e20e5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 18:02:21 +0200
Subject: [PATCH 365/623] fix artificial test cases ('family' becomes mandatory
 in the action info, but dict was supplied in the test case)

---
 fail2ban/tests/servertestcase.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 603bb69f..060d8e40 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1074,16 +1074,16 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				action.start()
 				# test ban ip4 :
 				logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
-				action.ban({'ip': IPAddr('192.0.2.1')})
+				action.ban({'ip': IPAddr('192.0.2.1'), 'family': 'inet4'})
 				# test unban ip4 :
 				logSys.debug('# === unban ipv4 ==='); self.pruneLog()
-				action.unban({'ip': IPAddr('192.0.2.1')})
+				action.unban({'ip': IPAddr('192.0.2.1'), 'family': 'inet4'})
 				# test ban ip6 :
 				logSys.debug('# === ban ipv6 ==='); self.pruneLog()
-				action.ban({'ip': IPAddr('2001:DB8::')})
+				action.ban({'ip': IPAddr('2001:DB8::'), 'family': 'inet6'})
 				# test unban ip6 :
 				logSys.debug('# === unban ipv6 ==='); self.pruneLog()
-				action.unban({'ip': IPAddr('2001:DB8::')})
+				action.unban({'ip': IPAddr('2001:DB8::'), 'family': 'inet6'})
 				# test stop :
 				logSys.debug('# === stop ==='); self.pruneLog()
 				action.stop()

From daa13eb5ddbcd40750a4a8ee1e608341dd6005f5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 18:33:33 +0200
Subject: [PATCH 366/623] no cover for unreachable and abstract

---
 fail2ban/server/action.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 8afeea6e..9d518cfd 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -203,17 +203,17 @@ class ActionBase(object):
 		self._name = name
 		self._logSys = getLogger("fail2ban.%s" % self.__class__.__name__)
 
-	def start(self):
+	def start(self): # pragma: no cover - abstract
 		"""Executed when the jail/action is started.
 		"""
 		pass
 
-	def stop(self):
+	def stop(self): # pragma: no cover - abstract
 		"""Executed when the jail/action is stopped.
 		"""
 		pass
 
-	def ban(self, aInfo):
+	def ban(self, aInfo): # pragma: no cover - abstract
 		"""Executed when a ban occurs.
 
 		Parameters
@@ -224,7 +224,7 @@ class ActionBase(object):
 		"""
 		pass
 
-	def unban(self, aInfo):
+	def unban(self, aInfo): # pragma: no cover - abstract
 		"""Executed when a ban expires.
 
 		Parameters
@@ -399,7 +399,7 @@ class CommandAction(ActionBase):
 			# check the action depends on family (conditional):
 			if self._startOnDemand:
 				return True
-		elif self.__started.get(family):
+		elif self.__started.get(family): # pragma: no cover - normally unreachable
 			return True
 		return self._executeOperation('<actionstart>', 'starting', family=family)
 

From 44a26c615923f99526dc92e6efe67c1c6f8fe83e Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 29 Mar 2017 23:14:33 +0200
Subject: [PATCH 367/623] Update ChangeLog

amend to gh-1742
---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index bd49286e..da4854b6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,6 +29,7 @@ TODO: implementing of options resp. other tasks from PR #1346
   - fixed using new tag `<ip-host>` (without external command execution)
 * fail2ban-regex: fixed matched output by multi-line (buffered) parsing
 * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422)
+* fixed ipv6-action errors on systems not supporting ipv6 und umgekehrt (gh-1741)
 
 ### New Features
 * New Actions:
@@ -81,6 +82,12 @@ TODO: implementing of options resp. other tasks from PR #1346
     the parsing of log-entries contain new-line chars (as single entry);
   - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` 
     could be used in regex to enable it;
+* implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742):
+  - new action parameter `actionstart_on_demand` (bool) can be set to prevent/allow starting action
+    on demand (default retrieved automatically, if some conditional parameter `param?family=...` 
+    presents in action properties), see `action.d/pf.conf` for example;
+  - additionally `actionstop` will be executed only for families previously executing `actionstart`
+    (starting on demand only)
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From a1e9cc552c1f8c2afaed42954b58dc4afc12b70f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 23:05:52 +0200
Subject: [PATCH 368/623] bulk unban: introduced new command `actionflush`:
 executed in order to flush all bans at once (e. g. by unban all, reload with
 removing action, stop, shutdown the system); the actions having `actionflush`
 do not execute `actionunban` for each single ticket

---
 fail2ban/client/actionreader.py |  1 +
 fail2ban/server/action.py       | 21 +++++++++++++++++++++
 fail2ban/server/actions.py      | 18 +++++++++++++++---
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index b85f22a0..ace0b898 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -40,6 +40,7 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionstart": ["string", None],
 		"actionstart_on_demand": ["string", None],
 		"actionstop": ["string", None],
+		"actionflush": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 9d518cfd..d00458ba 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -281,6 +281,8 @@ class CommandAction(ActionBase):
 			self.actioncheck = ''
 			## Command executed in order to restore sane environment in error case.
 			self.actionrepair = ''
+			## Command executed in order to flush all bans at once (e. g. by stop/shutdown the system).
+			self.actionflush = ''
 			## Command executed in order to stop the system.
 			self.actionstop = ''
 			## Command executed in case of reloading action.
@@ -447,6 +449,25 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
+	def flush(self):
+		"""Executes the "actionflush" command.
+		
+		Command executed in order to flush all bans at once (e. g. by stop/shutdown 
+		the system), instead of unbunning of each single ticket.
+
+		Replaces the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		family = []
+		# cumulate started families, if started on demand (conditional):
+		if self._startOnDemand:
+			for f in CommandAction.COND_FAMILIES:
+				if self.__started.get(f) == 1: # only real started:
+					family.append(f)
+			# if no started (on demand) actions:
+			if not family: return True
+		return self._executeOperation('<actionflush>', 'flushing', family=family)
+
 	def stop(self):
 		"""Executes the "actionstop" command.
 
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 3a85e569..e652872e 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -447,25 +447,37 @@ class Actions(JailThread, Mapping):
 		If actions specified, don't flush list - just execute unban for 
 		given actions (reload, obsolete resp. removed actions).
 		"""
+		log = True
 		if actions is None:
 			logSys.debug("Flush ban list")
 			lst = self.__banManager.flushBanList()
 		else:
+			log = False
 			lst = iter(self.__banManager)
 		cnt = 0
+		# first we'll execute flush for actions supporting this operation:
+		unbactions = {}
+		for name, action in (actions if actions is not None else self._actions).iteritems():
+			if hasattr(action, 'flush') and action.actionflush:
+				logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name)
+				action.flush()
+			else:
+				unbactions[name] = action
+		actions = unbactions
+		# unban each ticket with non-flasheable actions:
 		for ticket in lst:
 			# delete ip from database also:
 			if db and self._jail.database is not None:
 				ip = str(ticket.getIP())
 				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
-			self.__unBan(ticket, actions=actions)
+			self.__unBan(ticket, actions=actions, log=log)
 			cnt += 1
 		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
 			cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __unBan(self, ticket, actions=None):
+	def __unBan(self, ticket, actions=None, log=True):
 		"""Unbans host corresponding to the ticket.
 
 		Executes the actions in order to unban the host given in the
@@ -482,7 +494,7 @@ class Actions(JailThread, Mapping):
 			unbactions = actions
 		ip = ticket.getIP()
 		aInfo = self.__getActionInfo(ticket)
-		if actions is None:
+		if log:
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
 			try:

From d03872fbbf1b78d6047f4bc1876e2acc3293873d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 23:20:43 +0200
Subject: [PATCH 369/623] bulk unban: add new command `actionflush` default for
 several iptables/iptables-ipset actions (and common include):  
 iptables-common   iptables   iptables-allports   iptables-multiport-log  
 iptables-multiport   iptables-new   iptables-ipset-proto4  
 iptables-ipset-proto6   iptables-ipset-proto6-allports

executing `actionflush` command covered for this actions now
---
 config/action.d/iptables-allports.conf        |  2 +-
 config/action.d/iptables-common.conf          |  8 +++++
 config/action.d/iptables-ipset-proto4.conf    |  9 +++++-
 .../iptables-ipset-proto6-allports.conf       |  8 ++++-
 config/action.d/iptables-ipset-proto6.conf    |  8 ++++-
 config/action.d/iptables-multiport-log.conf   | 10 +++++--
 config/action.d/iptables-multiport.conf       |  2 +-
 config/action.d/iptables-new.conf             |  2 +-
 config/action.d/iptables-xt_recent-echo.conf  |  6 ++++
 config/action.d/iptables.conf                 |  2 +-
 fail2ban/tests/servertestcase.py              | 29 +++++++++++++++++++
 11 files changed, 77 insertions(+), 9 deletions(-)

diff --git a/config/action.d/iptables-allports.conf b/config/action.d/iptables-allports.conf
index 15f3cbcc..dbea5984 100644
--- a/config/action.d/iptables-allports.conf
+++ b/config/action.d/iptables-allports.conf
@@ -26,7 +26,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck
diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf
index a3921021..e016ef2f 100644
--- a/config/action.d/iptables-common.conf
+++ b/config/action.d/iptables-common.conf
@@ -16,6 +16,14 @@ after = iptables-blocktype.local
         iptables-common.local
 # iptables-blocktype.local is obsolete
 
+[Definition]
+
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = <iptables> -F f2b-<name>
+
 
 [Init]
 
diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf
index 2f63cd4b..30353f36 100644
--- a/config/action.d/iptables-ipset-proto4.conf
+++ b/config/action.d/iptables-ipset-proto4.conf
@@ -30,12 +30,19 @@ before = iptables-common.conf
 actionstart = ipset --create f2b-<name> iphash
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
 
+
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset --flush f2b-<name>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset --flush f2b-<name>
+             <actionflush>
              ipset --destroy f2b-<name>
 
 # Option:  actionban
diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf
index 113f599e..b761ad8c 100644
--- a/config/action.d/iptables-ipset-proto6-allports.conf
+++ b/config/action.d/iptables-ipset-proto6-allports.conf
@@ -29,12 +29,18 @@ before = iptables-common.conf
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset flush <ipmset>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>
-             ipset flush <ipmset>
+             <actionflush>
              ipset destroy <ipmset>
 
 # Option:  actionban
diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf
index dee7b029..e337eedf 100644
--- a/config/action.d/iptables-ipset-proto6.conf
+++ b/config/action.d/iptables-ipset-proto6.conf
@@ -29,12 +29,18 @@ before = iptables-common.conf
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset flush <ipmset>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
-             ipset flush <ipmset>
+             <actionflush>
              ipset destroy <ipmset>
 
 # Option:  actionban
diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf
index 1777ce62..62c2b4b1 100644
--- a/config/action.d/iptables-multiport-log.conf
+++ b/config/action.d/iptables-multiport-log.conf
@@ -26,13 +26,19 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I f2b-<name>-log -j LOG --log-prefix "$(expr f2b-<name> : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2
               <iptables> -A f2b-<name>-log -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = <iptables> -F f2b-<name>
+              <iptables> -F f2b-<name>-log
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
-             <iptables> -F f2b-<name>-log
+             <actionflush>
              <iptables> -X f2b-<name>
              <iptables> -X f2b-<name>-log
 
diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf
index 9fd87d20..c05f6ffc 100644
--- a/config/action.d/iptables-multiport.conf
+++ b/config/action.d/iptables-multiport.conf
@@ -23,7 +23,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck
diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf
index 795bc601..5b316807 100644
--- a/config/action.d/iptables-new.conf
+++ b/config/action.d/iptables-new.conf
@@ -25,7 +25,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck
diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf
index 018d2cf6..1970de14 100644
--- a/config/action.d/iptables-xt_recent-echo.conf
+++ b/config/action.d/iptables-xt_recent-echo.conf
@@ -35,6 +35,12 @@ before = iptables-common.conf
 #    shorter of the two timeouts actually matters.
 actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
+# Option:  actionflush
+#
+# [TODO] Flushing is currently not implemented for xt_recent
+#
+actionflush = 
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf
index 38985ffa..bf83e24a 100644
--- a/config/action.d/iptables.conf
+++ b/config/action.d/iptables.conf
@@ -23,7 +23,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> --dport <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 060d8e40..604a15ee 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1195,6 +1195,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-mp`",
+						"`ip6tables -w -F f2b-j-w-iptables-mp`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
 						"`iptables -w -F f2b-j-w-iptables-mp`",
@@ -1235,6 +1239,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-ap`",
+						"`ip6tables -w -F f2b-j-w-iptables-ap`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
 						"`iptables -w -F f2b-j-w-iptables-ap`",
@@ -1273,6 +1281,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
+					'flush': (
+						"`ipset flush f2b-j-w-iptables-ipset`",
+						"`ipset flush f2b-j-w-iptables-ipset6`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-iptables-ipset`",
@@ -1307,6 +1319,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
+					'flush': (
+						"`ipset flush f2b-j-w-iptables-ipset-ap`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap6`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-iptables-ipset-ap`",
@@ -1343,6 +1359,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables`",
+						"`ip6tables -w -F f2b-j-w-iptables`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
 						"`iptables -w -F f2b-j-w-iptables`",
@@ -1383,6 +1403,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
 						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-new`",
+						"`ip6tables -w -F f2b-j-w-iptables-new`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
 						"`iptables -w -F f2b-j-w-iptables-new`",
@@ -1684,6 +1708,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					action.unban(ainfo['ip6'])
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
+					# test flush for actions should supported this:
+					if tests.get('flush'):
+						self.pruneLog('# === flush ===')
+						action.flush()
+						self.assertLogged(*tests['flush'], all=True)
 					# test stop :
 					self.pruneLog('# === stop ===')
 					action.stop()

From 042a060a54ed517e91496073fb8f7ad66ee7a78e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 29 Mar 2017 23:23:32 +0200
Subject: [PATCH 370/623] additionally complex test-case coverage for
 `actionflush` inside server via actions-mechanism of fail2ban - reload with
 removing action, unban all, stopping of jails and actions, etc.

---
 fail2ban/tests/fail2banclienttestcase.py | 41 ++++++++++++++++++++----
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 35e00421..3e046042 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -762,6 +762,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"norestored = %(_exec_once)s",
 				"restore = ",
 				"info = ",
+				"_use_flush_ = echo [<name>] <actname>: -- flushing IPs",
 				"actionstart =  echo '[%(name)s] %(actname)s: ** start'", start,
 				"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
 				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s%(info)s'", ban,
@@ -788,7 +789,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 					if 1 in actions else "",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>',"
+										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
@@ -802,7 +804,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']"
+										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
@@ -874,6 +877,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Creating new jail 'test-jail2'",
 			"Jail 'test-jail2' started", all=True)
+		# test action3 removed, test flushing successful (and no single unban occurred):
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action3: -- flushing IPs'",
+			"stdout: '[test-jail1] test-action3: __ stop'", all=True)
+		self.assertNotLogged(
+			"stdout: '[test-jail1] test-action3: -- unban 192.0.2.1'")
 		
 		# update action1, delete action2 (should be stopped via configuration)...
 		self.pruneLog("[test-phase 2a]")
@@ -969,12 +978,18 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'",
 			all=True)
 
-		# don't need actions anymore:
-		_write_action_cfg(actname="test-action2", allow=False)
-		_write_jail_cfg(actions=[])
+		# ban manually to test later flush by unban all:
+		self.pruneLog("[test-phase 2d]")
+		self.execSuccess(startparams,
+			"set", "test-jail2", "banip", "192.0.2.21")
+		self.execSuccess(startparams,
+			"set", "test-jail2", "banip", "192.0.2.22")
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22 ", all=True, wait=MID_WAITTIME)
 
 		# restart jail with unban all:
-		self.pruneLog("[test-phase 2d]")
+		self.pruneLog("[test-phase 2e]")
 		self.execSuccess(startparams,
 			"restart", "--unban", "test-jail2")
 		self.assertLogged(
@@ -986,12 +1001,26 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Unban 192.0.2.4",
 			"[test-jail2] Unban 192.0.2.8", all=True
 		)
+		# test unban (action2):
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action2: -- unban 192.0.2.21",
+			"stdout: '[test-jail2] test-action2: -- unban 192.0.2.22'", all=True)
+		# test flush (action3, and no single unban via action3 occurred):
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action3: -- flushing IPs'")
+		self.assertNotLogged(
+			"stdout: '[test-jail2] test-action3: -- unban 192.0.2.21'",
+			"stdout: '[test-jail2] test-action3: -- unban 192.0.2.22'", all=True)
 		# no more ban (unbanned all):
 		self.assertNotLogged(
 			"[test-jail2] Ban 192.0.2.4",
 			"[test-jail2] Ban 192.0.2.8", all=True
 		)
 
+		# don't need actions anymore:
+		_write_action_cfg(actname="test-action2", allow=False)
+		_write_jail_cfg(actions=[])
+
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
 		self.execSuccess(startparams, "reload", "test-jail1")

From 97e8b42d342c437d27d43fa4ed136c11a5fa1be4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 30 Mar 2017 13:02:37 +0200
Subject: [PATCH 371/623] dummy action extended with more examples and
 test-covered now

---
 config/action.d/dummy.conf       | 26 +++++++++++++++++++++-----
 fail2ban/server/actions.py       |  2 +-
 fail2ban/tests/servertestcase.py | 27 +++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/config/action.d/dummy.conf b/config/action.d/dummy.conf
index dc4e1dbf..41250c27 100644
--- a/config/action.d/dummy.conf
+++ b/config/action.d/dummy.conf
@@ -10,14 +10,23 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = touch /var/run/fail2ban/fail2ban.dummy
-              printf %%b "<init>\n" >> /var/run/fail2ban/fail2ban.dummy
+actionstart = if [ ! -z '<target>' ]; then touch <target>; fi;
+              printf %%b "<init>\n" <to_target>
+              echo "%(debug)s started"
+
+# Option:  actionflush
+# Notes.:  command executed once to flush (clear) all IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = printf %%b "-*\n" <to_target>
+              echo "%(debug)s clear all"
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = rm -f /var/run/fail2ban/fail2ban.dummy
+actionstop = if [ ! -z '<target>' ]; then rm -f <target>; fi;
+             echo "%(debug)s stopped"
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -31,7 +40,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "+<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
+actionban = printf %%b "+<ip>\n" <to_target>
+            echo "%(debug)s banned <ip> (family: <family>)"
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -39,9 +49,15 @@ actionban = printf %%b "+<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = printf %%b "-<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
+actionunban = printf %%b "-<ip>\n" <to_target>
+              echo "%(debug)s unbanned <ip> (family: <family>)"
+
+
+debug = [<name>] <actname> <target> --
 
 [Init]
 
 init = 123
 
+target = /var/run/fail2ban/fail2ban.dummy
+to_target = >> <target>
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index e652872e..c33359c9 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -452,7 +452,7 @@ class Actions(JailThread, Mapping):
 			logSys.debug("Flush ban list")
 			lst = self.__banManager.flushBanList()
 		else:
-			log = False
+			log = False # don't log "[jail] Unban ..." if removing actions only.
 			lst = iter(self.__banManager)
 		cnt = 0
 		# first we'll execute flush for actions supporting this operation:
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 604a15ee..1644d895 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1182,6 +1182,33 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			#   'start', 'stop' - should be found (logged) on action start/stop,
 			#   etc.
 			testJailsActions = (
+				# dummy --
+				('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', {
+					'ip4': ('family: inet4',), 'ip6': ('family: inet6',),
+					'start': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`',
+					), 
+					'flush': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- clear all"`',
+					),
+					'stop': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- stopped"`',
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 192.0.2.1 (family: inet4)"`',
+					),
+					'ip4-unban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 192.0.2.1 (family: inet4)"`',
+					),
+					'ip6-ban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 2001:db8:: (family: inet6)"`',
+					),
+					'ip6-unban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 2001:db8:: (family: inet6)"`',
+					),					
+				}),
 				# iptables-multiport --
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),

From e7f1fc5cb360a2aebcb71c037ec32b0a889ab3f1 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 31 Mar 2017 10:39:50 +0200
Subject: [PATCH 372/623] Update ChangeLog

enhancements of #1743
---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index da4854b6..d644f330 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -88,6 +88,10 @@ TODO: implementing of options resp. other tasks from PR #1346
     presents in action properties), see `action.d/pf.conf` for example;
   - additionally `actionstop` will be executed only for families previously executing `actionstart`
     (starting on demand only)
+* introduced new command `actionflush`: executed in order to flush all bans at once 
+  e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
+  the actions having `actionflush` do not execute `actionunban` for each single ticket
+* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From 4fc6323ff038439e8b7d70ecf7d09488e0ac79a9 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Fri, 7 Apr 2017 13:59:22 +0200
Subject: [PATCH 373/623] haproxy-http-auth: avoid port number in IPv6
 addresses

The solution taken is to consume the port number explicitely in
the regexp.
---
 ChangeLog                                   | 1 +
 config/filter.d/haproxy-http-auth.conf      | 2 +-
 fail2ban/tests/files/logs/haproxy-http-auth | 2 ++
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d644f330..07a144f4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -23,6 +23,7 @@ TODO: implementing of options resp. other tasks from PR #1346
 * filter.d/sendmail-reject.conf:
   - rewritten using `prefregex` and used MLFID-related multi-line parsing;
   - optional parameter `mode` introduced: normal (default), extra or aggressive
+* filter.d/haproxy-http-auth: do not mistake client port for part of an IPv6 address (gh-1745)
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`
diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf
index 298ca292..d85f5e9e 100644
--- a/config/filter.d/haproxy-http-auth.conf
+++ b/config/filter.d/haproxy-http-auth.conf
@@ -28,7 +28,7 @@ _daemon = haproxy
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = ^%(__prefix_line)s<HOST>.*<NOSRV> -1/-1/-1/-1/\+*\d* 401
+failregex = ^%(__prefix_line)s<HOST>:\d+.*<NOSRV> -1/-1/-1/-1/\+*\d* 401
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth
index 298f1972..5a5141fc 100644
--- a/fail2ban/tests/files/logs/haproxy-http-auth
+++ b/fail2ban/tests/files/logs/haproxy-http-auth
@@ -2,3 +2,5 @@
 Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439] main app/app1 1939/0/1/0/1940 403 5168 - - ---- 3/3/0/0/0 0/0 "GET / HTTP/1.1"
 # failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" }
 Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/<NOSRV> -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1"
+# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "2001:db8::1234" }
+Nov 14 22:45:11 test haproxy[760]: 2001:db8::1234:58430 [14/Nov/2015:22:45:11.608] main main/<NOSRV> -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1"

From 07023436aca2a71ba338fe9f342c0908bd0b6e39 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Fri, 7 Apr 2017 14:04:09 +0200
Subject: [PATCH 374/623] haproxy-http-auth: added a test for
 IPv4-mapped-in-IPv6

This what one gets in logis if haproxy is binding to ::
on a dual-stack system.
---
 fail2ban/tests/files/logs/haproxy-http-auth | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth
index 5a5141fc..403a8083 100644
--- a/fail2ban/tests/files/logs/haproxy-http-auth
+++ b/fail2ban/tests/files/logs/haproxy-http-auth
@@ -4,3 +4,5 @@ Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439]
 Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/<NOSRV> -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1"
 # failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "2001:db8::1234" }
 Nov 14 22:45:11 test haproxy[760]: 2001:db8::1234:58430 [14/Nov/2015:22:45:11.608] main main/<NOSRV> -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1"
+# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" }
+Nov 14 22:45:11 test haproxy[760]: ::ffff:192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/<NOSRV> -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1"

From 4f0f22702ac795b53d8c953992e330c85e749142 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 11 Apr 2017 09:11:08 +0200
Subject: [PATCH 375/623] Update haproxy-http-auth.conf

little bit more precise expression
---
 config/filter.d/haproxy-http-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf
index d85f5e9e..f92f9d67 100644
--- a/config/filter.d/haproxy-http-auth.conf
+++ b/config/filter.d/haproxy-http-auth.conf
@@ -28,7 +28,7 @@ _daemon = haproxy
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = ^%(__prefix_line)s<HOST>:\d+.*<NOSRV> -1/-1/-1/-1/\+*\d* 401
+failregex = ^%(__prefix_line)s<HOST>(?::\d+)?\s+.*<NOSRV> -1/-1/-1/-1/\+*\d* 401
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

From bb79e7f41327983df068119ec48b90a5d0ceab47 Mon Sep 17 00:00:00 2001
From: Peter van der Does <peter@avirtualhome.com>
Date: Tue, 11 Apr 2017 11:13:58 -0400
Subject: [PATCH 376/623] Parameter not needed

The parameter '-s' causes an error as the <mailcmd> already has the parameter.
---
 config/action.d/mail-whois-lines.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index 0852ba8f..7ebb8b9f 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -21,7 +21,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: started on `uname -n`" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -30,7 +30,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command

From 17922b621c2d8dc99120470647ece16c56f298d8 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 20 Apr 2017 15:23:59 +0200
Subject: [PATCH 377/623] Update ChangeLog

replaced german in entry ;)
---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 07a144f4..17670805 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,7 +30,7 @@ TODO: implementing of options resp. other tasks from PR #1346
   - fixed using new tag `<ip-host>` (without external command execution)
 * fail2ban-regex: fixed matched output by multi-line (buffered) parsing
 * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422)
-* fixed ipv6-action errors on systems not supporting ipv6 und umgekehrt (gh-1741)
+* fixed ipv6-action errors on systems not supporting ipv6 and vice versa (gh-1741)
 
 ### New Features
 * New Actions:

From 6dfd080e20d1e088b5d840933aa864b88fb77d24 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 21 Apr 2017 11:17:13 +0200
Subject: [PATCH 378/623] Update apache-auth.conf

remove forgotten referer, that may prevent failure recognition (belongs to gh-1645)
---
 config/filter.d/apache-auth.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf
index f301e290..e22a49b7 100644
--- a/config/filter.d/apache-auth.conf
+++ b/config/filter.d/apache-auth.conf
@@ -21,8 +21,8 @@ failregex = ^client denied by server configuration\b
             ^Authorization of user <F-USER>(?:\S*|.*?)</F-USER> to access .*? failed\b
             ^%(auth_type)suser <F-USER>(?:\S*|.*?)</F-USER>: password mismatch\b
             ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (not found|denied by provider)\b
-            ^%(auth_type)sinvalid nonce .* received - length is not \S+(, referer: \S+)?\s*$
-            ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected `.+'(, referer: \S+)?\s*$
+            ^%(auth_type)sinvalid nonce .* received - length is not\b
+            ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected\b
             ^%(auth_type)sunknown algorithm `(?:[^']*|.*?)' received\b
             ^invalid qop `(?:[^']*|.*?)' received\b
             ^%(auth_type)sinvalid nonce .*? received - user attempted time travel\b

From e35ed1cdf7222a1dc1c8caaba08568b78d703a04 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 21 Apr 2017 11:24:32 +0200
Subject: [PATCH 379/623] Update ChangeLog

Changes of #1645
---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 17670805..9e722608 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -13,6 +13,8 @@ TODO: implementing of options resp. other tasks from PR #1346
       documentation should be extended (new options, etc)
       
 ### Fixes
+* `filter.d/apache-auth.conf`:
+  - better failure recognition using short form of regex (url/referer are foreign inputs, see gh-1645)
 * `filter.d/pam-generic.conf`:
   - [grave] injection on user name to host fixed
 * `filter.d/sshd.conf`:

From 507034c5be77c7894e2eee8afc2ce4ea84002674 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 24 Apr 2017 15:32:44 +0200
Subject: [PATCH 380/623] filter.d/apache-auth.conf: joined some similar
 expressions

---
 config/filter.d/apache-auth.conf | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf
index e22a49b7..d9a6fa5e 100644
--- a/config/filter.d/apache-auth.conf
+++ b/config/filter.d/apache-auth.conf
@@ -14,10 +14,8 @@ prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$
 # auth_type = ((?:Digest|Basic): )?
 auth_type = ([A-Z]\w+: )?
 
-failregex = ^client denied by server configuration\b
-            ^user <F-USER>(?:\S*|.*?)</F-USER> auth(?:oriz|entic)ation failure\b
-            ^user <F-USER>(?:\S*|.*?)</F-USER> not found\b
-            ^client used wrong authentication scheme\b
+failregex = ^client (?:denied by server configuration|used wrong authentication scheme)\b
+            ^user <F-USER>(?:\S*|.*?)</F-USER> (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b
             ^Authorization of user <F-USER>(?:\S*|.*?)</F-USER> to access .*? failed\b
             ^%(auth_type)suser <F-USER>(?:\S*|.*?)</F-USER>: password mismatch\b
             ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (not found|denied by provider)\b

From 99344d28c8fe258de7fb72810ad5646ab62dbbc7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 24 Apr 2017 20:17:57 +0200
Subject: [PATCH 381/623] Introduces new tags with hostname: - `<fq-hostname>`
 - fully-qualified name of host (the same as `$(hostname -f)`) -
 `<sh-hostname>` - short hostname (the same as `$(uname -n)`)

Execution of `uname -n` replaced in all mail actions with most interesting fully-qualified `<fq-hostname>`.
---
 config/action.d/mail-buffered.conf            |  6 +++---
 config/action.d/mail-whois-lines.conf         |  6 +++---
 config/action.d/mail-whois.conf               |  6 +++---
 config/action.d/mail.conf                     |  6 +++---
 config/action.d/sendmail-buffered.conf        |  8 +++----
 config/action.d/sendmail-common.conf          |  4 ++--
 config/action.d/sendmail-geoip-lines.conf     |  2 +-
 .../sendmail-whois-ipjailmatches.conf         |  2 +-
 config/action.d/sendmail-whois-ipmatches.conf |  2 +-
 config/action.d/sendmail-whois-lines.conf     |  2 +-
 config/action.d/sendmail-whois-matches.conf   |  2 +-
 config/action.d/sendmail-whois.conf           |  2 +-
 config/action.d/sendmail.conf                 |  2 +-
 config/action.d/xarf-login-attack.conf        |  4 ++--
 config/jail.conf                              |  2 +-
 fail2ban/server/actions.py                    |  4 ++++
 fail2ban/server/ipdns.py                      | 21 +++++++++++++++++++
 17 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf
index e74db9cc..88cd623f 100644
--- a/config/action.d/mail-buffered.conf
+++ b/config/action.d/mail-buffered.conf
@@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Output will be buffered until <lines> lines are available.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -28,13 +28,13 @@ actionstop = if [ -f <tmpfile> ]; then
                  These hosts have been banned by Fail2Ban.\n
                  `cat <tmpfile>`
                  Regards,\n
-                 Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from `uname -n`" <dest>
+                 Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest>
                  rm <tmpfile>
              fi
              printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index 7ebb8b9f..37e2d9b0 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -21,7 +21,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -30,7 +30,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -56,7 +56,7 @@ _ban_mail_content = ( printf %%b "Hi,\n
             Regards,\n
             Fail2Ban" )
 
-actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf
index 553bfb69..1f69f4c6 100644
--- a/config/action.d/mail-whois.conf
+++ b/config/action.d/mail-whois.conf
@@ -20,7 +20,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n
             Here is more information about <ip> :\n
             `%(_whois_command)s`\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>
+            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf
index 4715ecc5..cfc1cf65 100644
--- a/config/action.d/mail.conf
+++ b/config/action.d/mail.conf
@@ -16,7 +16,7 @@ norestored = 1
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on `uname -n`" <dest>
+              Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>
+            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf
index a91a6957..37bc642d 100644
--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@@ -17,7 +17,7 @@ norestored = 1
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
+actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
               From: <sendername> <<sender>>
               To: <dest>\n
               Hi,\n
@@ -31,7 +31,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
 # Values:  CMD
 #
 actionstop = if [ -f <tmpfile> ]; then
-                 printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
+                 printf %%b "Subject: [Fail2Ban] <name>: summary from <fq-hostname>
                  From: <sendername> <<sender>>
                  To: <dest>\n
                  Hi,\n
@@ -41,7 +41,7 @@ actionstop = if [ -f <tmpfile> ]; then
                  Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
                  rm <tmpfile>
              fi
-             printf %%b "Subject: [Fail2Ban] <name>: stopped  on `uname -n`
+             printf %%b "Subject: [Fail2Ban] <name>: stopped  on <fq-hostname>
              From: Fail2Ban <<sender>>
              To: <dest>\n
              Hi,\n
@@ -64,7 +64,7 @@ actioncheck =
 actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
-                printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
+                printf %%b "Subject: [Fail2Ban] <name>: summary from <fq-hostname>
                 From: <sendername> <<sender>>
                 To: <dest>\n
                 Hi,\n
diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf
index 1475dedb..46eca9ca 100644
--- a/config/action.d/sendmail-common.conf
+++ b/config/action.d/sendmail-common.conf
@@ -14,7 +14,7 @@ after = sendmail-common.local
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
+actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
               Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
               From: <sendername> <<sender>>
               To: <dest>\n
@@ -27,7 +27,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on `uname -n`
+actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on <fq-hostname>
              Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
              From: <sendername> <<sender>>
              To: <dest>\n
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index decf2c05..b7c1bf36 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -23,7 +23,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf
index 5bcefe89..06ea3a3e 100644
--- a/config/action.d/sendmail-whois-ipjailmatches.conf
+++ b/config/action.d/sendmail-whois-ipjailmatches.conf
@@ -19,7 +19,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf
index 4a8edcb7..83bff1b4 100644
--- a/config/action.d/sendmail-whois-ipmatches.conf
+++ b/config/action.d/sendmail-whois-ipmatches.conf
@@ -19,7 +19,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index e3a1c974..4b947cb2 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -20,7 +20,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf
index fc4ba061..01520135 100644
--- a/config/action.d/sendmail-whois-matches.conf
+++ b/config/action.d/sendmail-whois-matches.conf
@@ -19,7 +19,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf
index b8d99423..2fb01ed3 100644
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@@ -19,7 +19,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf
index 62c94439..cf420915 100644
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@@ -19,7 +19,7 @@ norestored = 1
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostname>
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 5274cdaf..9d441aa3 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -46,7 +46,7 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(di
             FROM=<sender>
             SERVICE=<service>
             FAILURES=<failures>
-            REPORTID=<time>@`uname -n`
+            REPORTID=<time>@<fq-hostname>
             TLP=<tlp>
             PORT=<port>
             DATE=`LC_ALL=C date --date=@<time> +"%%a, %%d %%h %%Y %%T %%z"`
@@ -119,7 +119,7 @@ logpath = /dev/null
 
 # Option:  sender
 # Notes.:  This is the sender that is included in the XARF report
-sender = fail2ban@`uname -n`
+sender = fail2ban@<fq-hostname>
 
 # Option:  port
 # Notes.:  This is the port number that received the login-attack
diff --git a/config/jail.conf b/config/jail.conf
index 7e5cc9b7..135c9a2b 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -134,7 +134,7 @@ filter = %(__name__)s
 destemail = root@localhost
 
 # Sender email address used solely for some actions
-sender = root@localhost
+sender = root@<fq-hostname>
 
 # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the
 # mailing. Change mta configuration parameter to mail if you want to
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index c33359c9..f8f8d4e1 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -35,6 +35,7 @@ except ImportError:
 	OrderedDict = dict
 
 from .banmanager import BanManager
+from .ipdns import DNSUtils
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
@@ -306,6 +307,9 @@ class Actions(JailThread, Mapping):
 			"ipjailmatches":	lambda self: "\n".join(self._mi4ip().getMatches()),
 			"ipfailures":			lambda self: self._mi4ip(True).getAttempt(),
 			"ipjailfailures":	lambda self: self._mi4ip().getAttempt(),
+			# system-information:
+			"fq-hostname":	lambda self: DNSUtils.getHostname(fqdn=True),
+			"sh-hostname":	lambda self: DNSUtils.getHostname(fqdn=False)
 		}
 
 		__slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip')
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index bda32ae8..e1aa1920 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -118,6 +118,27 @@ class DNSUtils:
 
 		return ipList
 
+	@staticmethod
+	def getHostname(fqdn=True):
+		"""Get short hostname or fully-qualified hostname of host self"""
+		# try find cached own hostnames (this tuple-key cannot be used elsewhere):
+		key = ('self','hostname', fqdn)
+		name = DNSUtils.CACHE_ipToName.get(key)
+		# get it using different ways (hostname, fully-qualified or vice versa):
+		if name is None:
+			name = ''
+			for hostname in (
+				(socket.getfqdn, socket.gethostname) if fqdn else (socket.gethostname, socket.getfqdn)
+			):
+				try:
+					name = hostname()
+					break
+				except Exception as e: # pragma: no cover
+					logSys.warning("Retrieving own hostnames failed: %s", e)
+		# cache and return :
+		DNSUtils.CACHE_ipToName.set(key, name)
+		return name
+
 	@staticmethod
 	def getSelfNames():
 		"""Get own host names of self"""

From db16b4c22add97b619d24d5e1a1fa5bb60b4cdd8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 24 Apr 2017 20:54:08 +0200
Subject: [PATCH 382/623] testComplexMailActionMultiLog: better echo mock-up
 replacement for "...) | mail -s ..."

---
 fail2ban/tests/servertestcase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 1644d895..32f794a6 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1748,7 +1748,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 		def _executeMailCmd(self, realCmd, timeout=60):
 			# replace pipe to mail with pipe to cat:
 			realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)',
-				r' echo mail \1 ) | cat', realCmd)
+				r') | cat; printf "\\n... | "; echo mail \1', realCmd)
 			# replace abuse retrieving (possible no-network), just replace first occurrence of 'dig...':
 			realCmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+',
 				lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"',

From f75c3d8a0243550b440393cab40077e74ced7da2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 24 Apr 2017 21:12:01 +0200
Subject: [PATCH 383/623] code review and ChangeLog entry

---
 ChangeLog                | 2 ++
 fail2ban/server/ipdns.py | 7 ++-----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9e722608..1978bf8b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -69,6 +69,8 @@ TODO: implementing of options resp. other tasks from PR #1346
   - `<ip-rev>` - PTR reversed representation of IP address
   - `<ip-host>` - host name of the IP address
   - `<F-...>` - interpolates to the corresponding filter group capture `...`
+  - `<fq-hostname>` - fully-qualified name of host (the same as `$(hostname -f)`)
+  - `<sh-hostname>` - short hostname (the same as `$(uname -n)`)
 * Allow to use filter options by `fail2ban-regex`, example:
   fail2ban-regex text.log "sshd[mode=aggressive]"
 * Samples test case factory extended with filter options - dict in JSON to control 
diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index e1aa1920..567a7f7d 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -148,11 +148,8 @@ class DNSUtils:
 		# get it using different ways (a set with names of localhost, hostname, fully qualified):
 		if names is None:
 			names = set(['localhost'])
-			for hostname in (socket.gethostname, socket.getfqdn):
-				try:
-					names |= set([hostname()])
-				except Exception as e: # pragma: no cover
-					logSys.warning("Retrieving own hostnames failed: %s", e)
+			for fqdn in (False, True):
+				names.add(DNSUtils.getHostname(fqdn=fqdn))
 		# cache and return :
 		DNSUtils.CACHE_ipToName.set(key, names)
 		return names

From e4ccbe728633fd12b90e1cf60a13de961ac53442 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 24 Apr 2017 21:32:40 +0200
Subject: [PATCH 384/623] code review: getHostname can return ''

---
 fail2ban/server/ipdns.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 567a7f7d..6ef36888 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -147,9 +147,9 @@ class DNSUtils:
 		names = DNSUtils.CACHE_ipToName.get(key)
 		# get it using different ways (a set with names of localhost, hostname, fully qualified):
 		if names is None:
-			names = set(['localhost'])
-			for fqdn in (False, True):
-				names.add(DNSUtils.getHostname(fqdn=fqdn))
+			names = set([
+				'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True)
+			]) - set(['']) # getHostname can return ''
 		# cache and return :
 		DNSUtils.CACHE_ipToName.set(key, names)
 		return names

From c3426ba5f6ecf6b2ac918c8e5579c96179169c06 Mon Sep 17 00:00:00 2001
From: Seth Reeser <seth.reeser@devopsgroup.io>
Date: Wed, 26 Apr 2017 14:14:39 -0400
Subject: [PATCH 385/623] Update botsearch-common.conf (#1759)

* Update botsearch-common.conf, apache-modsecurity.conf: typo and missing new-line
---
 config/filter.d/apache-modsecurity.conf | 2 +-
 config/filter.d/botsearch-common.conf   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/apache-modsecurity.conf b/config/filter.d/apache-modsecurity.conf
index 13e9c5ea..e296227a 100644
--- a/config/filter.d/apache-modsecurity.conf
+++ b/config/filter.d/apache-modsecurity.conf
@@ -16,4 +16,4 @@ ignoreregex =
 
 # https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats
 # Author: Daniel Black
-#         Sergey G. Brester aka sebres (review, optimization)
\ No newline at end of file
+#         Sergey G. Brester aka sebres (review, optimization)
diff --git a/config/filter.d/botsearch-common.conf b/config/filter.d/botsearch-common.conf
index a673a310..6052fab1 100644
--- a/config/filter.d/botsearch-common.conf
+++ b/config/filter.d/botsearch-common.conf
@@ -5,7 +5,7 @@
 # Block is the actual non-found directories to block
 block = \/?(<webmail>|<phpmyadmin>|<wordpress>|cgi-bin|mysqladmin)[^,]*
 
-# These are just convient definitions that assist the blocking of stuff that 
+# These are just convenient definitions that assist the blocking of stuff that 
 # isn't installed
 webmail = roundcube|(ext)?mail|horde|(v-?)?webmail
 

From 4bb8a58dcf19ba8cdef9af6fd81b5da3bb3490ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Fri, 28 Apr 2017 15:49:59 +0200
Subject: [PATCH 386/623] Courier may complain about the method only

> Mar 30 22:29:18 szerver imapd-ssl: LOGIN FAILED, method=PLAIN, ip=[::ffff:1.2.3.4]
---
 config/filter.d/courier-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/courier-auth.conf b/config/filter.d/courier-auth.conf
index 87ee55b4..9df74994 100644
--- a/config/filter.d/courier-auth.conf
+++ b/config/filter.d/courier-auth.conf
@@ -11,7 +11,7 @@ before = common.conf
 
 _daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)?
 
-failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$
+failregex = ^%(__prefix_line)sLOGIN FAILED, (user|method)=.*, ip=\[<HOST>\]$
 
 ignoreregex = 
 

From ac256a822b5ecc2758fabe86ec643363f3b2913a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Fri, 28 Apr 2017 16:58:24 +0200
Subject: [PATCH 387/623] Make courier-auth regexp a non-captured group

---
 config/filter.d/courier-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/courier-auth.conf b/config/filter.d/courier-auth.conf
index 9df74994..1ac33736 100644
--- a/config/filter.d/courier-auth.conf
+++ b/config/filter.d/courier-auth.conf
@@ -11,7 +11,7 @@ before = common.conf
 
 _daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)?
 
-failregex = ^%(__prefix_line)sLOGIN FAILED, (user|method)=.*, ip=\[<HOST>\]$
+failregex = ^%(__prefix_line)sLOGIN FAILED, (?:user|method)=.*, ip=\[<HOST>\]$
 
 ignoreregex = 
 

From 1ed958521c4bd6ba54816a4a80136218e7c78457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Fri, 28 Apr 2017 17:08:36 +0200
Subject: [PATCH 388/623] Courier auth changelog

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 2c3e91ab..c1921da4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -97,6 +97,7 @@ TODO: implementing of options resp. other tasks from PR #1346
   e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
   the actions having `actionflush` do not execute `actionunban` for each single ticket
 * add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
+* support Courier failed logins with method only
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From eb5c75e4b5c73dc126da72a18827eaf029bd813f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Sat, 29 Apr 2017 17:12:15 +0200
Subject: [PATCH 389/623] Add test to Courier auth

---
 fail2ban/tests/files/logs/courier-auth | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/courier-auth b/fail2ban/tests/files/logs/courier-auth
index e3d0d8c3..3505e109 100644
--- a/fail2ban/tests/files/logs/courier-auth
+++ b/fail2ban/tests/files/logs/courier-auth
@@ -6,3 +6,5 @@ Apr 23 21:59:38 dns2 pop3d: LOGIN FAILED, user=info@example.com, ip=[::ffff:198.
 Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff:198.51.100.33]
 # failJSON: { "time": "2005-04-17T19:17:11", "match": true , "host": "1.2.3.4" }
 Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4]
+# failJSON: { "time": "2005-04-17T19:17:12", "match": true , "host": "192.0.2.4" }
+Apr 17 19:17:12 server imapd-ssl: LOGIN FAILED, method=PLAIN, ip=[::ffff:192.0.2.4]

From b5d59e888377cb33cda7607f0a670c2930f82924 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 2 May 2017 16:59:05 +0200
Subject: [PATCH 390/623] small fix of changelog entry

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c1921da4..d5044590 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -97,7 +97,7 @@ TODO: implementing of options resp. other tasks from PR #1346
   e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
   the actions having `actionflush` do not execute `actionunban` for each single ticket
 * add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
-* support Courier failed logins with method only
+* filter.d/courier-auth.conf: support failed logins with method only
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc

From 2a4dafce5b8b583f2a914dd38d6add8bd4addd16 Mon Sep 17 00:00:00 2001
From: Sander Hoentjen <shoentjen@antagonist.nl>
Date: Mon, 8 May 2017 14:11:07 +0200
Subject: [PATCH 391/623] don't call setLogTarget from setSyslogSocket when
 holding the lock

If we do, setLogTarget will wait for the lock, but it will never be released
---
 fail2ban/server/server.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index facbe393..56367ed7 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -638,9 +638,9 @@ class Server:
 			if self.__syslogSocket == syslogsocket:
 				return True
 			self.__syslogSocket = syslogsocket
-			# Conditionally reload, logtarget depends on socket path when SYSLOG
-			return self.__logTarget != "SYSLOG"\
-				   or self.setLogTarget(self.__logTarget)
+		# Conditionally reload, logtarget depends on socket path when SYSLOG
+		return self.__logTarget != "SYSLOG"\
+			   or self.setLogTarget(self.__logTarget)
 
 	def getLogTarget(self):
 		with self.__loggingLock:

From a5cdb9c977fb640df2a12b3110235c2472042456 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 May 2017 09:10:45 +0200
Subject: [PATCH 392/623] exim test cases extended: cover short form of the
 logging (without session-id, gh-1771)

---
 fail2ban/tests/files/logs/exim | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index 326cdf28..8f3ac1bb 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -60,6 +60,8 @@
 2016-03-21 04:07:49 [25874] 1ahr79-0006jK-G9 SMTP connection from (voyeur.webair.com) [174.137.147.204]:44884 I=[172.89.0.6]:25 closed by DROP in ACL
 # failJSON: { "time": "2016-03-21T04:33:13", "match": true , "host": "206.214.71.53" }
 2016-03-21 04:33:13 [26074] 1ahrVl-0006mY-79 SMTP connection from riveruse.com [206.214.71.53]:39865 I=[172.89.0.6]:25 closed by DROP in ACL
+# failJSON: { "time": "2016-03-21T04:33:14", "match": true , "host": "192.0.2.33", "desc": "short form without optional session-id" }
+2016-03-21 04:33:14 SMTP connection from (some.domain) [192.0.2.33] closed by DROP in ACL
 
 # failJSON: { "time": "2016-04-01T11:08:39", "match": true , "host": "192.0.2.1" }
 2016-04-01 11:08:39 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.1]:1418 I=[172.89.0.6]:25 D=34s C=EHLO,AUTH

From 407b2ea936b836cbe9cdb59fc8e40fcce6d2959e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 11 May 2017 11:17:27 -0400
Subject: [PATCH 393/623] life is going on

---
 ChangeLog           | 15 +++++++++++++--
 README.md           |  6 +++---
 RELEASE             | 14 +++++++-------
 fail2ban/version.py |  2 +-
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ec855ab0..8e7ebdb0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,14 +6,25 @@
 Fail2Ban: Changelog
 ===================
 
-
-ver. 0.9.7 (2017/05/11) - awaiting-victory
+ver. 0.9.8 (2016/XX/XXX) - wanna-be-released
 -----------
 
 0.9.x line is no longer heavily developed.  If you are interested in
 new features (e.g. IPv6 support), please consider 0.10 branch and its
 releases.
 
+
+### Fixes
+
+### New Features
+
+### Enhancements
+
+
+
+ver. 0.9.7 (2017/05/11) - awaiting-victory
+-----------
+
 ### Fixes
 * Fixed a systemd-journal handling in fail2ban-regex (gh-1657)
 * filter.d/sshd.conf
diff --git a/README.md b/README.md
index 89844d1d..1138a3ac 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.9.7                  2017/05/11
+                       v0.9.7.dev0             2017/??/??
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
@@ -39,8 +39,8 @@ Optional:
 
 To install, just do:
 
-    tar xvfj fail2ban-0.9.6.tar.bz2
-    cd fail2ban-0.9.6
+    tar xvfj fail2ban-0.9.7.tar.bz2
+    cd fail2ban-0.9.7
     python setup.py install
 
 This will install Fail2Ban into the python library directory. The executable
diff --git a/RELEASE b/RELEASE
index c4f62d7a..0a3e0497 100644
--- a/RELEASE
+++ b/RELEASE
@@ -53,7 +53,7 @@ Preparation
 
   or an alternative for comparison with previous release
 
-     git diff 0.9.6 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST
+     git diff 0.9.7 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST
      sort MANIFEST | uniq | sponge MANIFEST
 
 * Run::
@@ -66,24 +66,24 @@ Preparation
 
   * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory::
 
-      tar -C /tmp -jxf dist/fail2ban-0.9.6.tar.bz2
+      tar -C /tmp -jxf dist/fail2ban-0.9.7.tar.bz2
 
 * clean up current directory::
 
-    diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.6/
+    diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.7/
 
   * Only differences should be files that you don't want distributed.
 
 
 * Ensure the tests work from the tarball::
 
-    cd /tmp/fail2ban-0.9.6/ && bin/fail2ban-testcases
+    cd /tmp/fail2ban-0.9.7/ && bin/fail2ban-testcases
 
 * Add/finalize the corresponding entry in the ChangeLog
 
   * To generate a list of committers use e.g.::
 
-      git shortlog -sn 0.9.6.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g'
+      git shortlog -sn 0.9.7.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g'
 
   * Ensure the top of the ChangeLog has the right version and current date.
   * Ensure the top entry of the ChangeLog has the right version and current date.
@@ -106,7 +106,7 @@ Preparation
 * Tag the release by using a signed (and annotated) tag.  Cut/paste
   release ChangeLog entry as tag annotation::
 
-    git tag -s 0.9.6
+    git tag -s 0.9.8
 
 Pre Release
 ===========
@@ -190,7 +190,7 @@ Post Release
 
 Add the following to the top of the ChangeLog::
 
-  ver. 0.9.8 (2016/XX/XXX) - wanna-be-released
+  ver. 0.9.9 (2016/XX/XXX) - wanna-be-released
   -----------
   
   ### Fixes
diff --git a/fail2ban/version.py b/fail2ban/version.py
index e8344517..3f4edc54 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.9.7"
+version = "0.9.7.dev0"

From e340d0d2b285dbc7b46c10beed9d1ea1ba04b09b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 May 2017 16:51:08 +0200
Subject: [PATCH 394/623] Fixed detection of directory-based log-rotation of
 pyinotify backend. If directory moved and the target is not watched path, so
 the monitoring of it could not be continued.

Now fixed with pending files await a monitoring if there (resp. its directories) appears again (respawn).

Closes gh-1769
---
 fail2ban/server/filter.py          |  3 +-
 fail2ban/server/filterpyinotify.py | 92 +++++++++++++++++++++++++++---
 fail2ban/tests/filtertestcase.py   | 46 ++++++++++++---
 3 files changed, 124 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 526f54ea..c88a6c6d 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -895,7 +895,8 @@ class FileFilter(Filter):
 			# see http://python.org/dev/peps/pep-3151/
 			except IOError as e:
 				logSys.error("Unable to open %s", filename)
-				logSys.exception(e)
+				if e.errno != 2:
+					logSys.exception(e)
 				return False
 			except OSError as e: # pragma: no cover - requires race condition to tigger this
 				logSys.error("Error opening %s", filename)
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 73c82099..faf01560 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -25,13 +25,14 @@ __license__ = "GPL"
 
 import logging
 from distutils.version import LooseVersion
+import os
 from os.path import dirname, sep as pathsep
 
 import pyinotify
 
 from .failmanager import FailManagerEmpty
 from .filter import FileFilter
-from .mytime import MyTime
+from .mytime import MyTime, time
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -52,6 +53,11 @@ except Exception as e:
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+# Override pyinotify default logger/init-handler:
+def _pyinotify_logger_init():
+	return logSys
+pyinotify._logger_init = _pyinotify_logger_init
+pyinotify.log = logSys
 
 ##
 # Log reader class.
@@ -73,6 +79,9 @@ class FilterPyinotify(FileFilter):
 		# Pyinotify watch manager
 		self.__monitor = pyinotify.WatchManager()
 		self.__watches = dict()
+		self.__pending = dict()
+		self.__pendingChkTime = 0
+		self.__pendingNextTime = 0
 		logSys.debug("Created FilterPyinotify")
 
 	def callback(self, event, origin=''):
@@ -84,15 +93,36 @@ class FilterPyinotify(FileFilter):
 				logSys.debug("Ignoring creation of directory %s", path)
 				return
 			# check if that is a file we care about
-			if not path in self.__watches:
+			if path not in self.__watches:
 				logSys.debug("Ignoring creation of %s we do not monitor", path)
 				return
-			else:
-				# we need to substitute the watcher with a new one, so first
-				# remove old one
-				self._delFileWatcher(path)
-				# place a new one
-				self._addFileWatcher(path)
+			self._refreshFileWatcher(path)
+		elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF):
+			# fix pyinotify behavior with '-unknown-path' (if target not watched also):
+			if (event.mask & pyinotify.IN_MOVE_SELF and path not in self.__watches and 
+					path.endswith('-unknown-path')
+			):
+				path = path[:-len('-unknown-path')]
+			# watch was removed for some reasons (log-rotate?):
+			if not os.path.isfile(path):
+				for log in self.getLogs():
+					logpath = log.getFileName()
+					if logpath.startswith(path):
+						# check exists (rotated):
+						if event.mask & pyinotify.IN_MOVE_SELF or not os.path.isfile(logpath):
+							self._addPendingFile(logpath, event)
+						else:
+							path = logpath
+							break
+			if path not in self.__watches:
+				logSys.debug("Ignoring event of %s we do not monitor", path)
+				return
+			if not os.path.isfile(path):
+				if self.containsLogPath(path):
+					self._addPendingFile(path, event)
+				logSys.debug("Ignoring watching/rotation event (%s) for %s", event.maskname, path)
+				return
+			self._refreshFileWatcher(path)
 		# do nothing if idle:
 		if self.idle:
 			return
@@ -113,6 +143,44 @@ class FilterPyinotify(FileFilter):
 			self.failManager.cleanup(MyTime.time())
 		self.__modified = False
 
+	def _addPendingFile(self, path, event):
+		if path not in self.__pending:
+			self.__pending[path] = self.sleeptime / 10;
+			logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s",
+				path, event.maskname, event.pathname)
+
+	def _checkPendingFiles(self):
+		if self.__pending:
+			ntm = time.time()
+			if ntm > self.__pendingNextTime:
+				found = {}
+				minTime = 60
+				for path, retardTM in self.__pending.iteritems():
+					if ntm - self.__pendingChkTime > retardTM:
+						if not os.path.isfile(path): # not found - prolong for next time
+							if retardTM < 60: retardTM *= 2
+							if minTime > retardTM: minTime = retardTM
+							self.__pending[path] = retardTM
+							continue
+						found[path] = 1
+						self._refreshFileWatcher(path)
+				for path in found:
+					try:
+						del self.__pending[path]
+					except KeyError: pass
+				self.__pendingChkTime = time.time()
+				self.__pendingNextTime = self.__pendingChkTime + minTime
+				# process now because we'he missed it in monitoring:
+				for path in found:
+					self._process_file(path)
+
+	def _refreshFileWatcher(self, oldPath, newPath=None):
+		# we need to substitute the watcher with a new one, so first
+		# remove old one
+		self._delFileWatcher(oldPath)
+		# place a new one
+		self._addFileWatcher(newPath or oldPath)
+
 	def _addFileWatcher(self, path):
 		wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY)
 		self.__watches.update(wd)
@@ -139,7 +207,9 @@ class FilterPyinotify(FileFilter):
 		if not (path_dir in self.__watches):
 			# we need to watch also  the directory for IN_CREATE
 			self.__watches.update(
-				self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO))
+				self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | 
+					pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF |
+					pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR))
 			logSys.debug("Added monitor for the parent directory %s", path_dir)
 
 		self._addFileWatcher(path)
@@ -177,6 +247,9 @@ class FilterPyinotify(FileFilter):
 
 	# slow check events while idle:
 	def __check_events(self, *args, **kwargs):
+		# check pending files (logrotate ready):
+		self._checkPendingFiles()
+
 		if self.idle:
 			if Utils.wait_for(lambda: not self.active or not self.idle,
 				self.sleeptime * 10, self.sleeptime
@@ -209,6 +282,7 @@ class FilterPyinotify(FileFilter):
 		super(FilterPyinotify, self).stop()
 		# Stop the notifier thread
 		self.__notifier.stop()
+		self.__notifier.stop = lambda *args: 0; # prevent dual stop
 
 	##
 	# Wait for exit with cleanup.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ce665e72..a9b31fd4 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -43,7 +43,7 @@ from ..server.failmanager import FailManagerEmpty
 from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
 from ..server.utils import Utils, uni_decode
-from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
+from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase
 from .dummyjail import DummyJail
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
@@ -942,17 +942,21 @@ def get_monitor_failures_testcase(Filter_):
 												  skip=3, mode='w')
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 
-		def test_move_file(self):
-			# if we move file into a new location while it has been open already
-			self.file.close()
-			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
-												  n=14, mode='w')
+		def _wait4failures(self, count=2):
 			# Poll might need more time
 			self.assertTrue(self.isEmpty(_maxWaitTime(5)),
 							"Queue must be empty but it is not: %s."
 							% (', '.join([str(x) for x in self.jail.queue])))
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
-			Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, _maxWaitTime(10))
+			Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10))
+			self.assertEqual(self.filter.failManager.getFailTotal(), count)
+
+		def test_move_file(self):
+			# if we move file into a new location while it has been open already
+			self.file.close()
+			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
+												  n=14, mode='w')
+			self._wait4failures()
 			self.assertEqual(self.filter.failManager.getFailTotal(), 2)
 
 			# move aside, but leaving the handle still open...
@@ -967,6 +971,34 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)
 
+		@with_tmpdir
+		def test_move_dir(self, tmp):
+			self.file.close()
+			self.filter.delLogPath(self.name)
+			# if we rename parent dir into a new location (simulate directory-base log rotation)
+			tmpsub1 = os.path.join(tmp, "1")
+			tmpsub2 = os.path.join(tmp, "2")
+			os.mkdir(tmpsub1)
+			self.name = os.path.join(tmpsub1, os.path.basename(self.name))
+			os.close(os.open(self.name, os.O_CREAT|os.O_APPEND)); # create empty file
+			self.filter.addLogPath(self.name, autoSeek=False)
+			
+			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
+												  skip=12, n=1, mode='w')
+			self.file.close()
+			self._wait4failures(1)
+
+			# rotate whole directory: rename directory 1 as 2:
+			os.rename(tmpsub1, tmpsub2)
+			os.mkdir(tmpsub1)
+			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
+												  skip=12, n=1, mode='w')
+			self.file.close()
+			self._wait4failures(2)
+			# stop before tmpdir deleted (just prevents many monitor events)
+			self.filter.stop()
+
+
 		def _test_move_into_file(self, interim_kill=False):
 			# if we move a new file into the location of an old (monitored) file
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name,

From 7b614a7a15b001e0e0e34765143dd43ea1095139 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 12 May 2017 22:01:37 +0200
Subject: [PATCH 395/623] differentiate between watched directories and files
 (refreshing monitoring of files/dirs expected different flags for watcher)

---
 fail2ban/server/filterpyinotify.py | 168 ++++++++++++++++++-----------
 1 file changed, 104 insertions(+), 64 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index faf01560..6394ecef 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -78,7 +78,8 @@ class FilterPyinotify(FileFilter):
 		self.__modified = False
 		# Pyinotify watch manager
 		self.__monitor = pyinotify.WatchManager()
-		self.__watches = dict()
+		self.__watchFiles = dict()
+		self.__watchDirs = dict()
 		self.__pending = dict()
 		self.__pendingChkTime = 0
 		self.__pendingNextTime = 0
@@ -87,45 +88,56 @@ class FilterPyinotify(FileFilter):
 	def callback(self, event, origin=''):
 		logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event)
 		path = event.pathname
+		# check watching of this path:
+		isWF = isWD = False
+		if path in self.__watchDirs:
+			isWD = True
+		elif path in self.__watchFiles:
+			isWF = True
+		# fix pyinotify behavior with '-unknown-path' (if target not watched also):
+		if (event.mask & pyinotify.IN_MOVE_SELF and 
+				path.endswith('-unknown-path') and not isWF and not isWD
+		):
+			path = path[:-len('-unknown-path')]
+			isWD = path in self.__watchDirs
+		assumeNoDir = False
 		if event.mask & ( pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO ):
+			# refresh watched dir (may be expected):
+			if isWD:
+				self._refreshWatcher(path, isDir=True)
+				return
 			# skip directories altogether
 			if event.mask & pyinotify.IN_ISDIR:
 				logSys.debug("Ignoring creation of directory %s", path)
 				return
 			# check if that is a file we care about
-			if path not in self.__watches:
+			if not isWF:
 				logSys.debug("Ignoring creation of %s we do not monitor", path)
 				return
-			self._refreshFileWatcher(path)
+			self._refreshWatcher(path)
 		elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF):
-			# fix pyinotify behavior with '-unknown-path' (if target not watched also):
-			if (event.mask & pyinotify.IN_MOVE_SELF and path not in self.__watches and 
-					path.endswith('-unknown-path')
-			):
-				path = path[:-len('-unknown-path')]
 			# watch was removed for some reasons (log-rotate?):
-			if not os.path.isfile(path):
-				for log in self.getLogs():
-					logpath = log.getFileName()
-					if logpath.startswith(path):
-						# check exists (rotated):
-						if event.mask & pyinotify.IN_MOVE_SELF or not os.path.isfile(logpath):
-							self._addPendingFile(logpath, event)
-						else:
-							path = logpath
-							break
-			if path not in self.__watches:
-				logSys.debug("Ignoring event of %s we do not monitor", path)
-				return
-			if not os.path.isfile(path):
-				if self.containsLogPath(path):
-					self._addPendingFile(path, event)
-				logSys.debug("Ignoring watching/rotation event (%s) for %s", event.maskname, path)
-				return
-			self._refreshFileWatcher(path)
+			assumeNoDir = event.mask & (pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF)
+			if isWD and (assumeNoDir or not os.path.isdir(path)):
+				self._addPending(path, event, isDir=True)
+			elif not isWF:
+				for logpath in self.__watchDirs:
+					if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)):
+						self._addPending(logpath, event, isDir=True)
+		# pending file:
+		for logpath in self.__watchFiles:
+			if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isfile(logpath)):
+				self._addPending(logpath, event)
+		if isWF and not os.path.isfile(path):
+			self._addPending(path, event)
+			return
 		# do nothing if idle:
 		if self.idle:
 			return
+		# be sure we process a file:
+		if not isWF:
+			logSys.debug("Ignoring event (%s) of %s we do not monitor", event.maskname, path)
+			return
 		self._process_file(path)
 
 	def _process_file(self, path):
@@ -143,27 +155,36 @@ class FilterPyinotify(FileFilter):
 			self.failManager.cleanup(MyTime.time())
 		self.__modified = False
 
-	def _addPendingFile(self, path, event):
+	def _addPending(self, path, event, isDir=False):
 		if path not in self.__pending:
-			self.__pending[path] = self.sleeptime / 10;
+			self.__pending[path] = [self.sleeptime / 10, isDir];
+			self.__pendingNextTime = 0
 			logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s",
 				path, event.maskname, event.pathname)
 
-	def _checkPendingFiles(self):
+	def _delPending(self, path):
+		try:
+			del self.__pending[path]
+		except KeyError: pass
+
+	def _checkPending(self):
 		if self.__pending:
 			ntm = time.time()
 			if ntm > self.__pendingNextTime:
 				found = {}
 				minTime = 60
-				for path, retardTM in self.__pending.iteritems():
+				for path, (retardTM, isDir) in self.__pending.iteritems():
 					if ntm - self.__pendingChkTime > retardTM:
-						if not os.path.isfile(path): # not found - prolong for next time
+						chkpath = os.path.isdir if isDir else os.path.isfile
+						if not chkpath(path): # not found - prolong for next time
 							if retardTM < 60: retardTM *= 2
 							if minTime > retardTM: minTime = retardTM
-							self.__pending[path] = retardTM
+							self.__pending[path][0] = retardTM
 							continue
-						found[path] = 1
-						self._refreshFileWatcher(path)
+						logSys.log(logging.MSG, "Log presence detected for %s %s", 
+							"directory" if isDir else "file", path)
+						found[path] = isDir
+						self._refreshWatcher(path, isDir=isDir)
 				for path in found:
 					try:
 						del self.__pending[path]
@@ -171,24 +192,32 @@ class FilterPyinotify(FileFilter):
 				self.__pendingChkTime = time.time()
 				self.__pendingNextTime = self.__pendingChkTime + minTime
 				# process now because we'he missed it in monitoring:
-				for path in found:
-					self._process_file(path)
+				for path, isDir in found.iteritems():
+					if not isDir:
+						self._process_file(path)
 
-	def _refreshFileWatcher(self, oldPath, newPath=None):
+	def _refreshWatcher(self, oldPath, newPath=None, isDir=False):
+		if not newPath: newPath = oldPath
 		# we need to substitute the watcher with a new one, so first
-		# remove old one
-		self._delFileWatcher(oldPath)
-		# place a new one
-		self._addFileWatcher(newPath or oldPath)
+		# remove old one and then place a new one
+		if not isDir:
+			self._delFileWatcher(oldPath)
+			self._addFileWatcher(newPath)
+		else:
+			self._delDirWatcher(oldPath)
+			self._addDirWatcher(newPath)
 
 	def _addFileWatcher(self, path):
+		# we need to watch also the directory for IN_CREATE
+		self._addDirWatcher(dirname(path))
+		# add file watcher:
 		wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY)
-		self.__watches.update(wd)
+		self.__watchFiles.update(wd)
 		logSys.debug("Added file watcher for %s", path)
 
 	def _delFileWatcher(self, path):
 		try:
-			wdInt = self.__watches.pop(path)
+			wdInt = self.__watchFiles.pop(path)
 			wd = self.__monitor.rm_watch(wdInt)
 			if wd[wdInt]:
 				logSys.debug("Removed file watcher for %s", path)
@@ -197,21 +226,30 @@ class FilterPyinotify(FileFilter):
 			pass
 		return False
 
+	def _addDirWatcher(self, path_dir):
+		# Add watch for the directory:
+		if path_dir not in self.__watchDirs:
+			self.__watchDirs.update(
+				self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | 
+					pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF |
+					pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR))
+			logSys.debug("Added monitor for the parent directory %s", path_dir)
+
+	def _delDirWatcher(self, path_dir):
+		# Remove watches for the directory:
+		try:
+			wdInt = self.__watchDirs.pop(path_dir)
+			self.__monitor.rm_watch(wdInt)
+		except KeyError: # pragma: no cover
+			pass
+		logSys.debug("Removed monitor for the parent directory %s", path_dir)
+
 	##
 	# Add a log file path
 	#
 	# @param path log file path
 
 	def _addLogPath(self, path):
-		path_dir = dirname(path)
-		if not (path_dir in self.__watches):
-			# we need to watch also  the directory for IN_CREATE
-			self.__watches.update(
-				self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | 
-					pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF |
-					pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR))
-			logSys.debug("Added monitor for the parent directory %s", path_dir)
-
 		self._addFileWatcher(path)
 		self._process_file(path)
 
@@ -223,18 +261,18 @@ class FilterPyinotify(FileFilter):
 	def _delLogPath(self, path):
 		if not self._delFileWatcher(path):
 			logSys.error("Failed to remove watch on path: %s", path)
+		self._delPending(path)
 
 		path_dir = dirname(path)
-		if not len([k for k in self.__watches
-					if k.startswith(path_dir + pathsep)]):
+		for k in self.__watchFiles:
+			if k.startswith(path_dir + pathsep):
+				path_dir = None
+				break
+		if path_dir:
 			# Remove watches for the directory
 			# since there is no other monitored file under this directory
-			try:
-				wdInt = self.__watches.pop(path_dir)
-				self.__monitor.rm_watch(wdInt)
-			except KeyError: # pragma: no cover
-				pass
-			logSys.debug("Removed monitor for the parent directory %s", path_dir)
+			self._delDirWatcher(path_dir)
+			self._delPending(path_dir)
 
 	# pyinotify.ProcessEvent default handler:
 	def __process_default(self, event):
@@ -247,14 +285,16 @@ class FilterPyinotify(FileFilter):
 
 	# slow check events while idle:
 	def __check_events(self, *args, **kwargs):
-		# check pending files (logrotate ready):
-		self._checkPendingFiles()
-
 		if self.idle:
 			if Utils.wait_for(lambda: not self.active or not self.idle,
 				self.sleeptime * 10, self.sleeptime
 			):
 				pass
+
+		# check pending files/dirs (logrotate ready):
+		if not self.idle:
+			self._checkPending()
+
 		self.ticks += 1
 		return pyinotify.ThreadedNotifier.check_events(self.__notifier, *args, **kwargs)
 

From 9841fe52c34c966d143e17e447268fce4e896214 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 12:52:16 +0200
Subject: [PATCH 396/623] fixed cleanup for Gamin backend (by interim stop in
 the test-cases)

---
 fail2ban/server/filtergamin.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py
index 106e4c0f..3baf8c54 100644
--- a/fail2ban/server/filtergamin.py
+++ b/fail2ban/server/filtergamin.py
@@ -143,6 +143,8 @@ class FilterGamin(FileFilter):
 	# Desallocates the resources used by Gamin.
 
 	def __cleanup(self):
+		if not self.monitor:
+			return
 		for filename in self.getLogPaths():
 			self.monitor.stop_watch(filename)
 		self.monitor = None

From 5c1d01bf58635f10dc2a5c6c273c2a8b983b0ee8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 14:31:18 +0200
Subject: [PATCH 397/623] code review, try to make recognition of pending files
 fewer sporadic (error prone)

---
 fail2ban/server/filterpyinotify.py | 50 +++++++++++++++++-------------
 fail2ban/tests/filtertestcase.py   |  1 -
 2 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 6394ecef..4926f9b7 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -38,7 +38,7 @@ from ..helpers import getLogger
 
 
 if not hasattr(pyinotify, '__version__') \
-  or LooseVersion(pyinotify.__version__) < '0.8.3':
+  or LooseVersion(pyinotify.__version__) < '0.8.3': # pragma: no cover
   raise ImportError("Fail2Ban requires pyinotify >= 0.8.3")
 
 # Verify that pyinotify is functional on this system
@@ -46,7 +46,7 @@ if not hasattr(pyinotify, '__version__') \
 try:
 	manager = pyinotify.WatchManager()
 	del manager
-except Exception as e:
+except Exception as e: # pragma: no cover
 	raise ImportError("Pyinotify is probably not functional on this system: %s"
 					  % str(e))
 
@@ -54,7 +54,7 @@ except Exception as e:
 logSys = getLogger(__name__)
 
 # Override pyinotify default logger/init-handler:
-def _pyinotify_logger_init():
+def _pyinotify_logger_init(): # pragma: no cover
 	return logSys
 pyinotify._logger_init = _pyinotify_logger_init
 pyinotify.log = logSys
@@ -124,10 +124,6 @@ class FilterPyinotify(FileFilter):
 				for logpath in self.__watchDirs:
 					if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)):
 						self._addPending(logpath, event, isDir=True)
-		# pending file:
-		for logpath in self.__watchFiles:
-			if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isfile(logpath)):
-				self._addPending(logpath, event)
 		if isWF and not os.path.isfile(path):
 			self._addPending(path, event)
 			return
@@ -155,12 +151,14 @@ class FilterPyinotify(FileFilter):
 			self.failManager.cleanup(MyTime.time())
 		self.__modified = False
 
-	def _addPending(self, path, event, isDir=False):
+	def _addPending(self, path, reason, isDir=False):
 		if path not in self.__pending:
 			self.__pending[path] = [self.sleeptime / 10, isDir];
 			self.__pendingNextTime = 0
+			if isinstance(reason, pyinotify.Event):
+				reason = [reason.maskname, reason.pathname]
 			logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s",
-				path, event.maskname, event.pathname)
+				path, *reason)
 
 	def _delPending(self, path):
 		try:
@@ -174,17 +172,18 @@ class FilterPyinotify(FileFilter):
 				found = {}
 				minTime = 60
 				for path, (retardTM, isDir) in self.__pending.iteritems():
-					if ntm - self.__pendingChkTime > retardTM:
-						chkpath = os.path.isdir if isDir else os.path.isfile
-						if not chkpath(path): # not found - prolong for next time
-							if retardTM < 60: retardTM *= 2
-							if minTime > retardTM: minTime = retardTM
-							self.__pending[path][0] = retardTM
-							continue
-						logSys.log(logging.MSG, "Log presence detected for %s %s", 
-							"directory" if isDir else "file", path)
-						found[path] = isDir
-						self._refreshWatcher(path, isDir=isDir)
+					if ntm - self.__pendingChkTime < retardTM:
+						if minTime > retardTM: minTime = retardTM
+						continue
+					chkpath = os.path.isdir if isDir else os.path.isfile
+					if not chkpath(path): # not found - prolong for next time
+						if retardTM < 60: retardTM *= 2
+						if minTime > retardTM: minTime = retardTM
+						self.__pending[path][0] = retardTM
+						continue
+					logSys.log(logging.MSG, "Log presence detected for %s %s", 
+						"directory" if isDir else "file", path)
+					found[path] = isDir
 				for path in found:
 					try:
 						del self.__pending[path]
@@ -193,7 +192,16 @@ class FilterPyinotify(FileFilter):
 				self.__pendingNextTime = self.__pendingChkTime + minTime
 				# process now because we'he missed it in monitoring:
 				for path, isDir in found.iteritems():
-					if not isDir:
+					self._refreshWatcher(path, isDir=isDir)
+					if isDir:
+						for logpath in self.__watchFiles:
+							if logpath.startswith(path + pathsep):
+								if not os.path.isfile(logpath):
+									self._addPending(logpath, ['FROM_PARDIR', path])
+								else:
+									self._refreshWatcher(logpath)
+									self._process_file(logpath)
+					else:
 						self._process_file(path)
 
 	def _refreshWatcher(self, oldPath, newPath=None, isDir=False):
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index a9b31fd4..ec2dea89 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -957,7 +957,6 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  n=14, mode='w')
 			self._wait4failures()
-			self.assertEqual(self.filter.failManager.getFailTotal(), 2)
 
 			# move aside, but leaving the handle still open...
 			os.rename(self.name, self.name + '.bak')

From ff1c6718da50f3d7eaf447bd399211f1fde7708b Mon Sep 17 00:00:00 2001
From: Filippo Tessarotto <zoeslam@gmail.com>
Date: Tue, 7 Feb 2017 12:58:04 +0100
Subject: [PATCH 398/623] Postfix RBL: 554 & SMTP

Cherry-pick of 607568f5da9e6b962c43d59280d72bd678c08afa (see gh-1686)
---
 config/filter.d/postfix-rbl.conf      | 2 +-
 fail2ban/tests/files/logs/postfix-rbl | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf
index c3f8c332..0a9078f0 100644
--- a/config/filter.d/postfix-rbl.conf
+++ b/config/filter.d/postfix-rbl.conf
@@ -12,7 +12,7 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/smtpd
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
 
 ignoreregex =
 
diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl
index eff01bf9..6aeac03b 100644
--- a/fail2ban/tests/files/logs/postfix-rbl
+++ b/fail2ban/tests/files/logs/postfix-rbl
@@ -3,3 +3,6 @@ Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.examp
 
 # failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
 Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
+
+# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" }
+Feb  7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from=<spammer@example.com> to=<goodguy@example.com> proto=SMTP helo=<WIN-5N8GBBS0R5I>

From 16a84ca0b5e5e5624e8d5871c356eb6d9c4c4b68 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 15:20:43 +0200
Subject: [PATCH 399/623] code review

---
 fail2ban/server/filter.py          |   2 +-
 fail2ban/server/filterpyinotify.py | 111 +++++++++++++++--------------
 2 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c88a6c6d..fce02a7a 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -895,7 +895,7 @@ class FileFilter(Filter):
 			# see http://python.org/dev/peps/pep-3151/
 			except IOError as e:
 				logSys.error("Unable to open %s", filename)
-				if e.errno != 2:
+				if e.errno != 2: # errno.ENOENT
 					logSys.exception(e)
 				return False
 			except OSError as e: # pragma: no cover - requires race condition to tigger this
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 4926f9b7..7785b84f 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -89,23 +89,12 @@ class FilterPyinotify(FileFilter):
 		logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event)
 		path = event.pathname
 		# check watching of this path:
-		isWF = isWD = False
-		if path in self.__watchDirs:
-			isWD = True
-		elif path in self.__watchFiles:
+		isWF = False
+		isWD = path in self.__watchDirs
+		if not isWD and path in self.__watchFiles:
 			isWF = True
-		# fix pyinotify behavior with '-unknown-path' (if target not watched also):
-		if (event.mask & pyinotify.IN_MOVE_SELF and 
-				path.endswith('-unknown-path') and not isWF and not isWD
-		):
-			path = path[:-len('-unknown-path')]
-			isWD = path in self.__watchDirs
 		assumeNoDir = False
 		if event.mask & ( pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO ):
-			# refresh watched dir (may be expected):
-			if isWD:
-				self._refreshWatcher(path, isDir=True)
-				return
 			# skip directories altogether
 			if event.mask & pyinotify.IN_ISDIR:
 				logSys.debug("Ignoring creation of directory %s", path)
@@ -116,8 +105,14 @@ class FilterPyinotify(FileFilter):
 				return
 			self._refreshWatcher(path)
 		elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF):
-			# watch was removed for some reasons (log-rotate?):
 			assumeNoDir = event.mask & (pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF)
+			# fix pyinotify behavior with '-unknown-path' (if target not watched also):
+			if (assumeNoDir and 
+					path.endswith('-unknown-path') and not isWF and not isWD
+			):
+				path = path[:-len('-unknown-path')]
+				isWD = path in self.__watchDirs
+			# watch was removed for some reasons (log-rotate?):
 			if isWD and (assumeNoDir or not os.path.isdir(path)):
 				self._addPending(path, event, isDir=True)
 			elif not isWF:
@@ -153,7 +148,7 @@ class FilterPyinotify(FileFilter):
 
 	def _addPending(self, path, reason, isDir=False):
 		if path not in self.__pending:
-			self.__pending[path] = [self.sleeptime / 10, isDir];
+			self.__pending[path] = [Utils.DEFAULT_SLEEP_INTERVAL, isDir];
 			self.__pendingNextTime = 0
 			if isinstance(reason, pyinotify.Event):
 				reason = [reason.maskname, reason.pathname]
@@ -166,43 +161,49 @@ class FilterPyinotify(FileFilter):
 		except KeyError: pass
 
 	def _checkPending(self):
-		if self.__pending:
-			ntm = time.time()
-			if ntm > self.__pendingNextTime:
-				found = {}
-				minTime = 60
-				for path, (retardTM, isDir) in self.__pending.iteritems():
-					if ntm - self.__pendingChkTime < retardTM:
-						if minTime > retardTM: minTime = retardTM
-						continue
-					chkpath = os.path.isdir if isDir else os.path.isfile
-					if not chkpath(path): # not found - prolong for next time
-						if retardTM < 60: retardTM *= 2
-						if minTime > retardTM: minTime = retardTM
-						self.__pending[path][0] = retardTM
-						continue
-					logSys.log(logging.MSG, "Log presence detected for %s %s", 
-						"directory" if isDir else "file", path)
-					found[path] = isDir
-				for path in found:
-					try:
-						del self.__pending[path]
-					except KeyError: pass
-				self.__pendingChkTime = time.time()
-				self.__pendingNextTime = self.__pendingChkTime + minTime
-				# process now because we'he missed it in monitoring:
-				for path, isDir in found.iteritems():
-					self._refreshWatcher(path, isDir=isDir)
-					if isDir:
-						for logpath in self.__watchFiles:
-							if logpath.startswith(path + pathsep):
-								if not os.path.isfile(logpath):
-									self._addPending(logpath, ['FROM_PARDIR', path])
-								else:
-									self._refreshWatcher(logpath)
-									self._process_file(logpath)
-					else:
-						self._process_file(path)
+		if not self.__pending:
+			return
+		ntm = time.time()
+		if ntm < self.__pendingNextTime:
+			return
+		found = {}
+		minTime = 60
+		for path, (retardTM, isDir) in self.__pending.iteritems():
+			if ntm - self.__pendingChkTime < retardTM:
+				if minTime > retardTM: minTime = retardTM
+				continue
+			chkpath = os.path.isdir if isDir else os.path.isfile
+			if not chkpath(path): # not found - prolong for next time
+				if retardTM < 60: retardTM *= 2
+				if minTime > retardTM: minTime = retardTM
+				self.__pending[path][0] = retardTM
+				continue
+			logSys.log(logging.MSG, "Log presence detected for %s %s", 
+				"directory" if isDir else "file", path)
+			found[path] = isDir
+		for path in found:
+			try:
+				del self.__pending[path]
+			except KeyError: pass
+		self.__pendingChkTime = time.time()
+		self.__pendingNextTime = self.__pendingChkTime + minTime
+		# process now because we've missed it in monitoring:
+		for path, isDir in found.iteritems():
+			# refresh monitoring of this:
+			self._refreshWatcher(path, isDir=isDir)
+			if isDir:
+				# check all files belong to this dir:
+				for logpath in self.__watchFiles:
+					if logpath.startswith(path + pathsep):
+						# if still no file - add to pending, otherwise refresh and process:
+						if not os.path.isfile(logpath):
+							self._addPending(logpath, ('FROM_PARDIR', path))
+						else:
+							self._refreshWatcher(logpath)
+							self._process_file(logpath)
+			else:
+				# process (possibly no old events for it from watcher):
+				self._process_file(path)
 
 	def _refreshWatcher(self, oldPath, newPath=None, isDir=False):
 		if not newPath: newPath = oldPath
@@ -286,7 +287,7 @@ class FilterPyinotify(FileFilter):
 	def __process_default(self, event):
 		try:
 			self.callback(event, origin='Default ')
-		except Exception as e:
+		except Exception as e: # pragma: no cover
 			logSys.error("Error in FilterPyinotify callback: %s",
 				e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
 		self.ticks += 1
@@ -300,7 +301,7 @@ class FilterPyinotify(FileFilter):
 				pass
 
 		# check pending files/dirs (logrotate ready):
-		if not self.idle:
+		if not self.idle and self.active:
 			self._checkPending()
 
 		self.ticks += 1

From f099558bcff64a65f8dc3e185838e1ce3c391576 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 15:43:16 +0200
Subject: [PATCH 400/623] try to fix travis build for pypy3 (after switch to
 'pypy3.3-5.2-alpha1' the test cases seems to never run anymore).

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 40376075..dc5b1e8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,8 +14,9 @@ python:
   # - pypy3
   - pypy3.3-5.2-alpha1
 before_install:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
+  - echo "running under $TRAVIS_PYTHON_VERSION"
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
   - travis_retry sudo apt-get update -qq
   # Set this so sudo executes the correct python binary
   #   Anything not using sudo will already have the correct environment

From 62e580b7cf647a2f66c19a833e5aa703ad1c19f8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 18:47:00 +0200
Subject: [PATCH 401/623] pyinotify: switch from ThreadedNotifier to Notifier:
 - Filter instance is already a thread; - avoid stop pyinotify processing if
 an interim error occurs (and breaks main-loop, e. g. during multi-threaded
 processing by add/remove log-files)

---
 fail2ban/server/filterpyinotify.py | 73 ++++++++++++++++++++----------
 1 file changed, 49 insertions(+), 24 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 7785b84f..29e29eca 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -290,23 +290,10 @@ class FilterPyinotify(FileFilter):
 		except Exception as e: # pragma: no cover
 			logSys.error("Error in FilterPyinotify callback: %s",
 				e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
+			# incr common error counter:
+			self.commonError()
 		self.ticks += 1
 
-	# slow check events while idle:
-	def __check_events(self, *args, **kwargs):
-		if self.idle:
-			if Utils.wait_for(lambda: not self.active or not self.idle,
-				self.sleeptime * 10, self.sleeptime
-			):
-				pass
-
-		# check pending files/dirs (logrotate ready):
-		if not self.idle and self.active:
-			self._checkPending()
-
-		self.ticks += 1
-		return pyinotify.ThreadedNotifier.check_events(self.__notifier, *args, **kwargs)
-
 	##
 	# Main loop.
 	#
@@ -317,26 +304,64 @@ class FilterPyinotify(FileFilter):
 		prcevent = pyinotify.ProcessEvent()
 		prcevent.process_default = self.__process_default
 		## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds)
-		self.__notifier = pyinotify.ThreadedNotifier(self.__monitor,
+		self.__notifier = pyinotify.Notifier(self.__monitor,
 			prcevent, timeout=self.sleeptime * 1000)
-		self.__notifier.check_events = self.__check_events
-		self.__notifier.start()
 		logSys.debug("[%s] filter started (pyinotifier)", self.jailName)
+		while self.active:
+			try:
+
+				# slow check events while idle:
+				if self.idle:
+					if Utils.wait_for(lambda: not self.active or not self.idle,
+						self.sleeptime * 10, self.sleeptime
+					):
+						if not self.active:
+							break
+
+				# default pyinotify handling using Notifier:
+				self.__notifier.process_events()
+				if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime):
+					if not self.active:
+						break
+					self.__notifier.read_events()
+
+				# check pending files/dirs (logrotate ready):
+				if not self.idle:
+					self._checkPending()
+
+			except Exception as e: # pragma: no cover
+				if not self.active: # if not active - error by stop...
+					break
+				logSys.error("Caught unhandled exception in main cycle: %r", e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+				# incr common error counter:
+				self.commonError()
+			
+			self.ticks += 1
+
+		logSys.debug("[%s] filter exited (pyinotifier)", self.jailName)
+		self.__notifier = None
+		
 		return True
 
 	##
 	# Call super.stop() and then stop the 'Notifier'
 
 	def stop(self):
+		if self.__notifier: # stop the notifier
+			self.__notifier.stop()
+		# stop filter thread:
 		super(FilterPyinotify, self).stop()
-		# Stop the notifier thread
-		self.__notifier.stop()
-		self.__notifier.stop = lambda *args: 0; # prevent dual stop
+		self.join()
+		if self.__notifier: # stop the notifier
+			self.__notifier.stop()
+			self.__notifier.stop = lambda *args: 0; # prevent dual stop
 
 	##
 	# Wait for exit with cleanup.
 
 	def join(self):
+		self.join = lambda *args: 0
 		self.__cleanup()
 		super(FilterPyinotify, self).join()
 		logSys.debug("[%s] filter terminated (pyinotifier)", self.jailName)
@@ -346,6 +371,6 @@ class FilterPyinotify(FileFilter):
 
 	def __cleanup(self):
 		if self.__notifier:
-			self.__notifier.join()			# to not exit before notifier does
-			self.__notifier = None
-		self.__monitor = None
+			if Utils.wait_for(lambda: not self.__notifier, self.sleeptime * 10):
+				self.__notifier = None
+				self.__monitor = None

From 050076ae42f13c74ae8ba418f5b1f40be96515cd Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 May 2017 19:05:40 +0200
Subject: [PATCH 402/623] code review + coverage fixes

---
 fail2ban/server/filterpyinotify.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 29e29eca..71540f3c 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -137,14 +137,15 @@ class FilterPyinotify(FileFilter):
 		TODO -- RF:
 		this is a common logic and must be shared/provided by FileFilter
 		"""
-		self.getFailures(path)
-		try:
-			while True:
-				ticket = self.failManager.toBan()
-				self.jail.putFailTicket(ticket)
-		except FailManagerEmpty:
-			self.failManager.cleanup(MyTime.time())
-		self.__modified = False
+		if not self.idle:
+			self.getFailures(path)
+			try:
+				while True:
+					ticket = self.failManager.toBan()
+					self.jail.putFailTicket(ticket)
+			except FailManagerEmpty:
+				self.failManager.cleanup(MyTime.time())
+			self.__modified = False
 
 	def _addPending(self, path, reason, isDir=False):
 		if path not in self.__pending:
@@ -268,7 +269,7 @@ class FilterPyinotify(FileFilter):
 	# @param path the log file to delete
 
 	def _delLogPath(self, path):
-		if not self._delFileWatcher(path):
+		if not self._delFileWatcher(path): # pragma: no cover
 			logSys.error("Failed to remove watch on path: %s", path)
 		self._delPending(path)
 
@@ -315,14 +316,12 @@ class FilterPyinotify(FileFilter):
 					if Utils.wait_for(lambda: not self.active or not self.idle,
 						self.sleeptime * 10, self.sleeptime
 					):
-						if not self.active:
-							break
+						if not self.active: break
 
 				# default pyinotify handling using Notifier:
 				self.__notifier.process_events()
 				if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime):
-					if not self.active:
-						break
+					if not self.active: break
 					self.__notifier.read_events()
 
 				# check pending files/dirs (logrotate ready):
@@ -341,7 +340,7 @@ class FilterPyinotify(FileFilter):
 
 		logSys.debug("[%s] filter exited (pyinotifier)", self.jailName)
 		self.__notifier = None
-		
+
 		return True
 
 	##
@@ -353,9 +352,6 @@ class FilterPyinotify(FileFilter):
 		# stop filter thread:
 		super(FilterPyinotify, self).stop()
 		self.join()
-		if self.__notifier: # stop the notifier
-			self.__notifier.stop()
-			self.__notifier.stop = lambda *args: 0; # prevent dual stop
 
 	##
 	# Wait for exit with cleanup.

From 17b0945a70e21395ebe2139ec18458155fe06433 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 16 May 2017 09:43:52 +0200
Subject: [PATCH 403/623] Update ChangeLog

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 78de41da..73df1383 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -33,6 +33,7 @@ TODO: implementing of options resp. other tasks from PR #1346
 * fail2ban-regex: fixed matched output by multi-line (buffered) parsing
 * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422)
 * fixed ipv6-action errors on systems not supporting ipv6 and vice versa (gh-1741)
+* fixed directory-based log-rotate for pyinotify-backend (gh-1778)
 
 ### New Features
 * New Actions:

From 9b83a3128d16fb545798283850a2760036915234 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 May 2017 12:23:28 +0200
Subject: [PATCH 404/623] code review, try to increase coverage

---
 fail2ban/server/filterpyinotify.py |  7 ++-----
 fail2ban/tests/filtertestcase.py   | 17 +++++++++++++++--
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 71540f3c..d945fc2d 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -115,7 +115,7 @@ class FilterPyinotify(FileFilter):
 			# watch was removed for some reasons (log-rotate?):
 			if isWD and (assumeNoDir or not os.path.isdir(path)):
 				self._addPending(path, event, isDir=True)
-			elif not isWF:
+			elif not isWF: # pragma: no cover (assume too sporadic)
 				for logpath in self.__watchDirs:
 					if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)):
 						self._addPending(logpath, event, isDir=True)
@@ -182,14 +182,11 @@ class FilterPyinotify(FileFilter):
 			logSys.log(logging.MSG, "Log presence detected for %s %s", 
 				"directory" if isDir else "file", path)
 			found[path] = isDir
-		for path in found:
-			try:
-				del self.__pending[path]
-			except KeyError: pass
 		self.__pendingChkTime = time.time()
 		self.__pendingNextTime = self.__pendingChkTime + minTime
 		# process now because we've missed it in monitoring:
 		for path, isDir in found.iteritems():
+			self._delPending(path)
 			# refresh monitoring of this:
 			self._refreshWatcher(path, isDir=isDir)
 			if isDir:
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ec2dea89..2bb64132 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -973,6 +973,7 @@ def get_monitor_failures_testcase(Filter_):
 		@with_tmpdir
 		def test_move_dir(self, tmp):
 			self.file.close()
+			self.filter.setMaxRetry(10)
 			self.filter.delLogPath(self.name)
 			# if we rename parent dir into a new location (simulate directory-base log rotation)
 			tmpsub1 = os.path.join(tmp, "1")
@@ -987,13 +988,25 @@ def get_monitor_failures_testcase(Filter_):
 			self.file.close()
 			self._wait4failures(1)
 
-			# rotate whole directory: rename directory 1 as 2:
-			os.rename(tmpsub1, tmpsub2)
+			# rotate whole directory: rename directory 1 as 2a:
+			os.rename(tmpsub1, tmpsub2 + 'a')
 			os.mkdir(tmpsub1)
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  skip=12, n=1, mode='w')
 			self.file.close()
 			self._wait4failures(2)
+
+			# rotate whole directory: rename directory 1 as 2b:
+			os.rename(tmpsub1, tmpsub2 + 'b')
+			# wait a bit in-between (try to increase coverage, should find pending file for pending dir):
+			self.waitForTicks(2)
+			os.mkdir(tmpsub1)
+			self.waitForTicks(2)
+			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
+												  skip=12, n=1, mode='w')
+			self.file.close()
+			self._wait4failures(3)
+
 			# stop before tmpdir deleted (just prevents many monitor events)
 			self.filter.stop()
 

From 96c3b06abbb11f7eb8735acd7741d62854e425ba Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 19 May 2017 13:25:49 +0200
Subject: [PATCH 405/623] amend to #1778: repair notifier wait-cycle (too long
 timeout in polling, too late check for pending files, too long stop)

---
 fail2ban/server/filterpyinotify.py | 32 ++++++++++++++++++++----------
 fail2ban/tests/filtertestcase.py   |  1 +
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 71540f3c..53f2cbda 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -82,7 +82,7 @@ class FilterPyinotify(FileFilter):
 		self.__watchDirs = dict()
 		self.__pending = dict()
 		self.__pendingChkTime = 0
-		self.__pendingNextTime = 0
+		self.__pendingMinTime = 60
 		logSys.debug("Created FilterPyinotify")
 
 	def callback(self, event, origin=''):
@@ -150,7 +150,7 @@ class FilterPyinotify(FileFilter):
 	def _addPending(self, path, reason, isDir=False):
 		if path not in self.__pending:
 			self.__pending[path] = [Utils.DEFAULT_SLEEP_INTERVAL, isDir];
-			self.__pendingNextTime = 0
+			self.__pendingMinTime = 0
 			if isinstance(reason, pyinotify.Event):
 				reason = [reason.maskname, reason.pathname]
 			logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s",
@@ -165,7 +165,7 @@ class FilterPyinotify(FileFilter):
 		if not self.__pending:
 			return
 		ntm = time.time()
-		if ntm < self.__pendingNextTime:
+		if ntm < self.__pendingChkTime + self.__pendingMinTime:
 			return
 		found = {}
 		minTime = 60
@@ -187,7 +187,7 @@ class FilterPyinotify(FileFilter):
 				del self.__pending[path]
 			except KeyError: pass
 		self.__pendingChkTime = time.time()
-		self.__pendingNextTime = self.__pendingChkTime + minTime
+		self.__pendingMinTime = minTime
 		# process now because we've missed it in monitoring:
 		for path, isDir in found.iteritems():
 			# refresh monitoring of this:
@@ -295,6 +295,12 @@ class FilterPyinotify(FileFilter):
 			self.commonError()
 		self.ticks += 1
 
+	@property
+	def __notify_maxtout(self):
+		# timeout for pyinotify must be set in milliseconds (fail2ban time values are 
+		# floats contain seconds), max 0.5 sec (additionally regards pending check time)
+		return min(self.sleeptime, 0.5, self.__pendingMinTime) * 1000
+
 	##
 	# Main loop.
 	#
@@ -304,9 +310,8 @@ class FilterPyinotify(FileFilter):
 	def run(self):
 		prcevent = pyinotify.ProcessEvent()
 		prcevent.process_default = self.__process_default
-		## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds)
 		self.__notifier = pyinotify.Notifier(self.__monitor,
-			prcevent, timeout=self.sleeptime * 1000)
+			prcevent, timeout=self.__notify_maxtout)
 		logSys.debug("[%s] filter started (pyinotifier)", self.jailName)
 		while self.active:
 			try:
@@ -314,13 +319,19 @@ class FilterPyinotify(FileFilter):
 				# slow check events while idle:
 				if self.idle:
 					if Utils.wait_for(lambda: not self.active or not self.idle,
-						self.sleeptime * 10, self.sleeptime
+						min(self.sleeptime * 10, self.__pendingMinTime), 
+						min(self.sleeptime, self.__pendingMinTime)
 					):
 						if not self.active: break
 
 				# default pyinotify handling using Notifier:
 				self.__notifier.process_events()
-				if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime):
+
+				# wait for events / timeout:
+				notify_maxtout = self.__notify_maxtout
+				def __check_events():
+					return not self.active or self.__notifier.check_events(timeout=notify_maxtout)
+				if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)):
 					if not self.active: break
 					self.__notifier.read_events()
 
@@ -347,11 +358,10 @@ class FilterPyinotify(FileFilter):
 	# Call super.stop() and then stop the 'Notifier'
 
 	def stop(self):
-		if self.__notifier: # stop the notifier
-			self.__notifier.stop()
 		# stop filter thread:
 		super(FilterPyinotify, self).stop()
-		self.join()
+		if self.__notifier: # stop the notifier
+			self.__notifier.stop()
 
 	##
 	# Wait for exit with cleanup.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ec2dea89..20a7ad50 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -996,6 +996,7 @@ def get_monitor_failures_testcase(Filter_):
 			self._wait4failures(2)
 			# stop before tmpdir deleted (just prevents many monitor events)
 			self.filter.stop()
+			self.filter.join()
 
 
 		def _test_move_into_file(self, interim_kill=False):

From c7ddf1f940aa30f7d31dc1bf6e4d895713a6e38b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 19 May 2017 15:36:06 +0200
Subject: [PATCH 406/623] [systemd-backend] implicit closing journal descriptor
 by stop filter. Partially cherry-picked from 0.10
 (d153555a07ec81dabe4678ea06c8e644afc8ec40)

---
 ChangeLog                        | 2 ++
 fail2ban/server/filtersystemd.py | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 8e7ebdb0..c798768d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,8 @@ releases.
 
 
 ### Fixes
+* Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991.
+  Partially back-ported from v.0.10.
 
 ### New Features
 
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 908112a7..73db618e 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -291,6 +291,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				except FailManagerEmpty:
 					self.failManager.cleanup(MyTime.time())
 
+		# close journal:
+		try:
+			if self.__journal:
+				self.__journal.close()
+		except Exception as e: # pragma: no cover
+			logSys.error("Close journal failed: %r", e,
+				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		logSys.debug((self.jail is not None and self.jail.name
                       or "jailless") +" filter terminated")
 		return True

From e8f217390449120a3c3c7fe4ca502b37eb8d2c56 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Tue, 23 May 2017 17:27:12 +0200
Subject: [PATCH 407/623] New logtimezone jail option

This new option allows to force the time zone on log lines
that don't bear a time zone indication (GitHub issue #1773), so it behaves
actually with respect to log line contents as a default time zone.

For the time being, only fixed offset timezones (UTC or UTC[+-]hhmm) are
supported, but the implementation is designed to later on treat the case
of logical timezones with DST, e.g., Europe/Paris etc.

In particular, the timezone name gets passed all the way to the strptime
module, and the resulting offset is computed for the given log line, even
though for now, it doesn't actually depend on it.

Also, the DateTemplate subclass gets to choose whether to use it or not.
For instance, it doesn't make sense to apply a time zone offset to
Unix timestamps.

The drawback is to introduce an API change for DateTemplate. I hope it's
internal enough for that not being a problem.
---
 fail2ban/client/jailreader.py          |  1 +
 fail2ban/server/datedetector.py        |  4 +--
 fail2ban/server/datetemplate.py        | 16 +++++++----
 fail2ban/server/filter.py              | 27 ++++++++++++++++--
 fail2ban/server/server.py              |  6 ++++
 fail2ban/server/strptime.py            | 39 ++++++++++++++++++++++++--
 fail2ban/server/transmitter.py         |  6 ++++
 fail2ban/tests/clientreadertestcase.py | 12 ++++++++
 fail2ban/tests/config/jail.conf        |  4 +++
 fail2ban/tests/datedetectortestcase.py | 15 ++++++++++
 fail2ban/tests/filtertestcase.py       | 10 +++++++
 fail2ban/tests/servertestcase.py       |  4 +++
 12 files changed, 133 insertions(+), 11 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index ca092990..ce0ed3b6 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -101,6 +101,7 @@ class JailReader(ConfigReader):
 				["string", "filter", ""]]
 		opts = [["bool", "enabled", False],
 				["string", "logpath", None],
+				["string", "logtimezone", None],
 				["string", "logencoding", None],
 				["string", "backend", "auto"],
 				["int",    "maxretry", None],
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index dd5d198f..39a15828 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -423,7 +423,7 @@ class DateDetector(object):
 		logSys.log(logLevel, " no template.")
 		return (None, None)
 
-	def getTime(self, line, timeMatch=None):
+	def getTime(self, line, timeMatch=None, default_tz=None):
 		"""Attempts to return the date on a log line using templates.
 
 		This uses the templates' `getDate` method in an attempt to find
@@ -449,7 +449,7 @@ class DateDetector(object):
 		template = timeMatch[1]
 		if template is not None:
 			try:
-				date = template.getDate(line, timeMatch[0])
+				date = template.getDate(line, timeMatch[0], default_tz=default_tz)
 				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
 						logSys.log(logLevel, "  got time %f for %r using template %s",
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 1d0b014b..44ea54a8 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -158,7 +158,7 @@ class DateTemplate(object):
 		return dateMatch
 
 	@abstractmethod
-	def getDate(self, line, dateMatch=None):
+	def getDate(self, line, dateMatch=None, default_tz=None):
 		"""Abstract method, which should return the date for a log line
 
 		This should return the date for a log line, typically taking the
@@ -169,6 +169,8 @@ class DateTemplate(object):
 		----------
 		line : str
 			Log line, of which the date should be extracted from.
+		default_tz: if no explicit time zone is present in the line
+                            passing this will interpret it as in that time zone.
 
 		Raises
 		------
@@ -200,13 +202,14 @@ class DateEpoch(DateTemplate):
 			regex = r"((?P<square>(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))"
 			self.setRegex(regex, wordBegin='start', wordEnd=True)
 
-	def getDate(self, line, dateMatch=None):
+	def getDate(self, line, dateMatch=None, default_tz=None):
 		"""Method to return the date for a log line.
 
 		Parameters
 		----------
 		line : str
 			Log line, of which the date should be extracted from.
+		default_tz: ignored, Unix timestamps are time zone independent
 
 		Returns
 		-------
@@ -277,7 +280,7 @@ class DatePatternRegex(DateTemplate):
 			regex = r'(?iu)' + regex
 		super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
 
-	def getDate(self, line, dateMatch=None):
+	def getDate(self, line, dateMatch=None, default_tz=None):
 		"""Method to return the date for a log line.
 
 		This uses a custom version of strptime, using the named groups
@@ -287,6 +290,7 @@ class DatePatternRegex(DateTemplate):
 		----------
 		line : str
 			Log line, of which the date should be extracted from.
+		default_tz: optionally used to correct timezone
 
 		Returns
 		-------
@@ -297,7 +301,8 @@ class DatePatternRegex(DateTemplate):
 		if not dateMatch:
 			dateMatch = self.matchDate(line)
 		if dateMatch:
-			return reGroupDictStrptime(dateMatch.groupdict()), dateMatch
+			return (reGroupDictStrptime(dateMatch.groupdict(), default_tz=default_tz),
+				dateMatch)
 
 
 class DateTai64n(DateTemplate):
@@ -315,13 +320,14 @@ class DateTai64n(DateTemplate):
 		# We already know the format for TAI64N
 		self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin)
 
-	def getDate(self, line, dateMatch=None):
+	def getDate(self, line, dateMatch=None, default_tz=None):
 		"""Method to return the date for a log line.
 
 		Parameters
 		----------
 		line : str
 			Log line, of which the date should be extracted from.
+		default_tz: ignored, since TAI is time zone independent
 
 		Returns
 		-------
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index fce02a7a..8d1eb856 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -40,6 +40,7 @@ from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
 from .utils import Utils
 from ..helpers import getLogger, PREFER_ENC
+from .strptime import validateTimeZone
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -102,6 +103,8 @@ class Filter(JailThread):
 		self.checkAllRegex = False
 		## if true ignores obsolete failures (failure time < now - findTime):
 		self.checkFindTime = True
+		## if set, treat log lines without explicit time zone to be in this time zone
+		self.logtimezone = None
 		## Ticks counter
 		self.ticks = 0
 
@@ -307,6 +310,22 @@ class Filter(JailThread):
 				return pattern, templates[0].name
 		return None
 
+	##
+	# Set the log default time zone
+	#
+	# @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM)
+
+	def setLogTimeZone(self, tz):
+		self.logtimezone = validateTimeZone(tz)
+
+	##
+	# Get the log default timezone
+	#
+	# @return symbolic timezone (a string)
+
+	def getLogTimeZone(self):
+		return self.logtimezone
+
 	##
 	# Set the maximum retry value.
 	#
@@ -621,7 +640,8 @@ class Filter(JailThread):
 			self.__lastDate = date
 		elif timeText:
 
-			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
+			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3],
+								  default_tz=self.logtimezone)
 
 			if dateTimeMatch is None:
 				logSys.error("findFailure failed to parse timeText: %s", timeText)
@@ -972,7 +992,10 @@ class FileFilter(Filter):
 					break
 				(timeMatch, template) = self.dateDetector.matchTime(line)
 				if timeMatch:
-					dateTimeMatch = self.dateDetector.getTime(line[timeMatch.start():timeMatch.end()], (timeMatch, template))
+					dateTimeMatch = self.dateDetector.getTime(
+						line[timeMatch.start():timeMatch.end()],
+						(timeMatch, template),
+						default_tz=self.logtimezone)
 				else:
 					nextp = container.tell()
 					if nextp > maxp:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 56367ed7..e3b22c44 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -379,6 +379,12 @@ class Server:
 	def getDatePattern(self, name):
 		return self.__jails[name].filter.getDatePattern()
 
+	def setLogTimeZone(self, name, tz):
+		self.__jails[name].filter.setLogTimeZone(tz)
+
+	def getLogTimeZone(self, name):
+		return self.__jails[name].filter.getLogTimeZone()
+
 	def setIgnoreCommand(self, name, value):
 		self.__jails[name].filter.setIgnoreCommand(value)
 
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 55bdcc8c..aff9db92 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -17,6 +17,7 @@
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+import re
 import time
 import calendar
 import datetime
@@ -26,6 +27,7 @@ from .mytime import MyTime
 
 locale_time = LocaleTime()
 timeRE = TimeRE()
+FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$')
 
 def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
 	""" Build century regex for last year and the next years (distance).
@@ -78,7 +80,36 @@ def getTimePatternRE():
 		names[key] = "%%%s" % key
 	return (patt, names)
 
-def reGroupDictStrptime(found_dict, msec=False):
+
+def validateTimeZone(tz):
+	"""Validate a timezone.
+
+	For now this accepts only the UTC[+-]hhmm format.
+	In the future, it may be extended for named time zones (such as Europe/Paris)
+        present on the system, if a suitable tz library is present.
+	"""
+	m = FIXED_OFFSET_TZ_RE.match(tz)
+	if m is None:
+		raise ValueError("Unknown or unsupported time zone: %r" % tz)
+	return tz
+
+def zone2offset(tz, dt):
+	"""Return the proper offset, in minutes according to given timezone at a given time.
+
+	Parameters
+	----------
+	tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have
+                               been validated already)
+        dt: datetime instance for offset computation
+	"""
+	if tz == 'UTC':
+		return 0
+	unsigned = int(tz[4:6])*60 + int(tz[6:])
+	if tz[3] == '-':
+		return -unsigned
+	return unsigned
+
+def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""Return time from dictionary of strptime fields
 
 	This is tweaked from python built-in _strptime.
@@ -88,7 +119,8 @@ def reGroupDictStrptime(found_dict, msec=False):
 	found_dict : dict
 		Dictionary where keys represent the strptime fields, and values the
 		respective value.
-
+	default_tz : default timezone to apply if nothing relevant is in found_dict
+                     (may be a non-fixed one in the future)
 	Returns
 	-------
 	float
@@ -209,6 +241,9 @@ def reGroupDictStrptime(found_dict, msec=False):
 	# Actully create date
 	date_result =  datetime.datetime(
 		year, month, day, hour, minute, second, fraction)
+	# Correct timezone if not supplied in the log linge
+	if tzoffset is None and default_tz is not None:
+		tzoffset = zone2offset(default_tz, date_result)
 	# Add timezone info
 	if tzoffset is not None:
 		date_result -= datetime.timedelta(seconds=tzoffset * 60)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index bc9edd43..ecc2a138 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -261,6 +261,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setDatePattern(name, value)
 			return self.__server.getDatePattern(name)
+		elif command[1] == "logtimezone":
+			value = command[2]
+			self.__server.setLogTimeZone(name, value)
+			return self.__server.getLogTimeZone(name)
 		elif command[1] == "maxretry":
 			value = command[2]
 			self.__server.setMaxRetry(name, int(value))
@@ -363,6 +367,8 @@ class Transmitter:
 			return self.__server.getFindTime(name)
 		elif command[1] == "datepattern":
 			return self.__server.getDatePattern(name)
+		elif command[1] == "logtimezone":
+			return self.__server.getLogTimeZone(name)
 		elif command[1] == "maxretry":
 			return self.__server.getMaxRetry(name)
 		elif command[1] == "maxlines":
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index bfa68e03..6cc2f659 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -196,6 +196,14 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertTrue(jail.isEnabled())
 		self.assertLogged("Invalid action definition 'joho[foo'")
 
+	def testJailLogTimeZone(self):
+		jail = JailReader('tz_correct', basedir=IMPERFECT_CONFIG,
+			share_config=IMPERFECT_CONFIG_SHARE_CFG)
+		self.assertTrue(jail.read())
+		self.assertTrue(jail.getOptions())
+		self.assertTrue(jail.isEnabled())
+		self.assertEqual(jail.options['logtimezone'], 'UTC+0200')
+
 	def testJailFilterBrokenDef(self):
 		jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG,
 			share_config=IMPERFECT_CONFIG_SHARE_CFG)
@@ -533,10 +541,14 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ]],
 			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
 			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
+			 ['set', 'tz_correct', 'addfailregex', '<IP>'],
+			 ['set', 'tz_correct', 'logtimezone', 'UTC+0200'],
 			 ['start', 'emptyaction'],
 			 ['start', 'missinglogfiles'],
 			 ['start', 'brokenaction'],
 			 ['start', 'parse_to_end_of_jail.conf'],
+		         ['add', 'tz_correct', 'auto'],
+			 ['start', 'tz_correct'],
 			 ['config-error',
 				"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"],
 			 ['config-error',
diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf
index 64c1b830..3dcbf634 100644
--- a/fail2ban/tests/config/jail.conf
+++ b/fail2ban/tests/config/jail.conf
@@ -47,3 +47,7 @@ action = thefunkychickendance
 [parse_to_end_of_jail.conf]
 enabled = true
 action =
+
+[tz_correct]
+enabled = true
+logtimezone = UTC+0200
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 39ab7173..16515f1b 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -89,6 +89,21 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertEqual(datelog, dateUnix)
 		self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
 
+	def testDefaultTimeZone(self):
+		log = "2017-01-23 15:00:00"
+		datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300')
+		# so in UTC, it was noon
+		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+				 datetime.datetime(2017, 1, 23, 12, 0, 0))
+
+		datelog, _ = self.datedetector.getTime(log, default_tz='UTC')
+		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+				 datetime.datetime(2017, 1, 23, 15, 0, 0))
+
+		datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430')
+		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+				 datetime.datetime(2017, 1, 23, 19, 30, 0))
+
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
 		"""
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 745dcca7..cb0edb06 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -289,6 +289,16 @@ class BasicFilter(unittest.TestCase):
 			("^%Y-%m-%d-%H%M%S.%f %z **",
 			"^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **"))
 
+	def testGetSetLogTimeZone(self):
+		self.assertEqual(self.filter.getLogTimeZone(), None)
+		self.filter.setLogTimeZone('UTC')
+		self.assertEqual(self.filter.getLogTimeZone(), 'UTC')
+		self.filter.setLogTimeZone('UTC-0400')
+		self.assertEqual(self.filter.getLogTimeZone(), 'UTC-0400')
+		self.filter.setLogTimeZone('UTC+0200')
+		self.assertEqual(self.filter.getLogTimeZone(), 'UTC+0200')
+		self.assertRaises(ValueError, self.filter.setLogTimeZone, 'not-a-time-zone')
+
 	def testAssertWrongTime(self):
 		self.assertRaises(AssertionError, 
 			lambda: _assert_equal_entries(self, 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 32f794a6..68b9951c 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -311,6 +311,10 @@ class Transmitter(TransmitterBase):
 			"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
 		self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
 
+	def testLogTimeZone(self):
+		self.setGetTest("logtimezone", "UTC+0400", "UTC+0400", jail=self.jailName)
+		self.setGetTestNOK("logtimezone", "not-a-time-zone", jail=self.jailName)
+
 	def testJailUseDNS(self):
 		self.setGetTest("usedns", "yes", jail=self.jailName)
 		self.setGetTest("usedns", "warn", jail=self.jailName)

From 5bb6be01636ca515ded2b906bb9c7f72ea85b96f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Tue, 30 May 2017 02:05:38 +0200
Subject: [PATCH 408/623] IPv6 address may overlap

---
 config/action.d/helpers-common.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index b04c7f7a..ccbcb172 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -5,7 +5,7 @@
 #   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
 #
 _grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
-_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
+_grep_logs_args = '(^|[^0-9:])<ip>([^0-9:]|$)'
 
 # Used for actions, that should not by executed if ticket was restored:
 _bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
@@ -13,4 +13,4 @@ _bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
 [Init]
 greplimit = tail -n <grepmax>
 grepmax = 1000
-grepopts = -m <grepmax>
\ No newline at end of file
+grepopts = -m <grepmax>

From 80cc47b75ff14858d9a7bab8410ce982745508da Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 30 May 2017 09:14:43 +0200
Subject: [PATCH 409/623] Update helpers-common.conf

fixed grep pattern: escape dot-char in search-IP and more restrictive boundaries (IPv6-capable)
---
 config/action.d/helpers-common.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index ccbcb172..5799d9d3 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -5,7 +5,7 @@
 #   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
 #
 _grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
-_grep_logs_args = '(^|[^0-9:])<ip>([^0-9:]|$)'
+_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '<ip>' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)"
 
 # Used for actions, that should not by executed if ticket was restored:
 _bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;

From 228d25c548a3f62d8606054709fcb29ff3367775 Mon Sep 17 00:00:00 2001
From: Marcel Bischoff <marcel@herrbischoff.com>
Date: Tue, 30 May 2017 20:27:44 +0200
Subject: [PATCH 410/623] Update Kerio Connect filter (#1455)

* Update Kerio Connect filter

Fixed regex for some log entries that did not get recognized and some additional error formats are added.

* Add missing colon, GitHub address

* Add filter tests

* Add missing test
---
 config/filter.d/kerio.conf      |  8 +++++++-
 fail2ban/tests/files/logs/kerio | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/kerio.conf b/config/filter.d/kerio.conf
index 313c9b36..ec1c09df 100644
--- a/config/filter.d/kerio.conf
+++ b/config/filter.d/kerio.conf
@@ -3,9 +3,14 @@
 [Definition]
 
 failregex = ^ SMTP Spam attack detected from <HOST>,
-            ^ IP address <HOST> found in DNS blacklist \S+, mail from \S+ to \S+$
+            ^ IP address <HOST> found in DNS blacklist
             ^ Relay attempt from IP address <HOST>
             ^ Attempt to deliver to unknown recipient \S+, from \S+, IP address <HOST>$
+            ^ Failed SMTP login from <HOST>
+            ^ SMTP: User \S+ doesn't exist. Attempt from IP address <HOST>
+            ^ Client with IP address <HOST> has no reverse DNS entry, connection rejected before SMTP greeting$
+            ^ Administration login into Web Administration from <HOST> failed: IP address not allowed$
+            ^ Message from IP address <HOST>, sender \S+ rejected: sender domain does not exist$
 
 ignoreregex =
 
@@ -16,5 +21,6 @@ datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
 # DEV NOTES:
 # 
 # Author: A.P. Lawrence
+# Updated by: M. Bischoff <https://github.com/herrbischoff>
 #
 # Based off: http://aplawrence.com/Kerio/fail2ban.html
diff --git a/fail2ban/tests/files/logs/kerio b/fail2ban/tests/files/logs/kerio
index c9368c22..f7aa0eb7 100644
--- a/fail2ban/tests/files/logs/kerio
+++ b/fail2ban/tests/files/logs/kerio
@@ -25,5 +25,20 @@
 # failJSON: { "time": "2013-12-13T01:11:04", "match": true, "host": "218.85.253.185" }
 [13/Dec/2013 01:11:04] Attempt to deliver to unknown recipient <marge@aplawrence.com>, from <yu@rrd.com>, IP address 218.85.253.185
 
+# failJSON: { "time": "2017-05-29T17:29:29", "match": true, "host": "185.140.108.56" }
+[29/May/2017 17:29:29] IP address 185.140.108.56 found in DNS blacklist SpamCop, mail from <noreply-tjgqNffcgPfpbZtpDzasm@oakspaversusa.com> to <info@verinion.com> rejected
 
+# failJSON: { "time": "2017-05-17T19:43:42", "match": true, "host": "185.140.108.26" }
+[17/May/2017 19:43:42] SMTP: User printer@verinion.com doesn't exist. Attempt from IP address 185.140.108.26.
 
+# failJSON: { "time": "2017-05-17T19:44:25", "match": true, "host": "184.171.168.211" }
+[17/May/2017 19:44:25] Client with IP address 184.171.168.211 has no reverse DNS entry, connection rejected before SMTP greeting
+
+# failJSON: { "time": "2017-05-17T19:45:27", "match": true, "host": "170.178.167.136" }
+[17/May/2017 19:45:27] Administration login into Web Administration from 170.178.167.136 failed: IP address not allowed
+
+# failJSON: { "time": "2017-05-17T22:14:57", "match": true, "host": "67.211.219.82" }
+[17/May/2017 22:14:57] Message from IP address 67.211.219.82, sender <promo123@goodresponse.site> rejected: sender domain does not exist
+
+# failJSON: { "time": "2017-05-18T07:25:15", "match": true, "host": "212.92.127.112" }
+[18/May/2017 07:25:15] Failed SMTP login from 212.92.127.112 with SASL method CRAM-MD5.

From 5214c1c5d1611e0c4a8f3c960a4b833ee767f7bc Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 30 May 2017 20:31:48 +0200
Subject: [PATCH 411/623] Update changelog (gh-1455)

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c798768d..5f472f81 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -21,7 +21,7 @@ releases.
 ### New Features
 
 ### Enhancements
-
+* filter.d/kerio.conf - filter extended with new rules (see gh-1455)
 
 
 ver. 0.9.7 (2017/05/11) - awaiting-victory

From b93e47b12f69419692fc8275358b498f01b30cbe Mon Sep 17 00:00:00 2001
From: Peter Nowee <peter+github@peternowee.com>
Date: Wed, 31 May 2017 15:24:21 +0200
Subject: [PATCH 412/623] dovecot: Match also when user field is empty

Commit 5678d08 of 2016-11-26 changed:

    ( user=<\S*>,)?

to:

    ( user=<[^>]+>,)?

The change from `*` (zero or more times) to `+` (one or more times) may
not have been intended. It will miss lines containing, for example:

    Aborted login (tried to use disallowed plaintext auth): user=<>

This commit reverts the `+` back to `*`.
---
 config/filter.d/dovecot.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index a3dc60bb..68c78c7a 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -10,7 +10,7 @@ before = common.conf
 _daemon = (auth|dovecot(-auth)?|auth-worker)
 
 failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
+            ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
             ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
             ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
             ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$

From 8cb4ae0242986a80e1d3eedd217fafbfeee082b7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 13:55:30 +0200
Subject: [PATCH 413/623] Code review and small optimizations, prepared to
 provide offset-based time zones for date-detectors (parsing of input-string)

---
 fail2ban/server/datedetector.py        | 15 ++++++++++++--
 fail2ban/server/filter.py              | 20 +++++++++----------
 fail2ban/server/strptime.py            | 27 ++++++++++++++------------
 fail2ban/tests/datedetectortestcase.py | 11 ++++++++---
 4 files changed, 46 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 39a15828..90cfe1fd 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -27,6 +27,7 @@ import time
 from threading import Lock
 
 from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .strptime import validateTimeZone
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -222,6 +223,8 @@ class DateDetector(object):
 		self.__firstUnused = 0
 		# pre-match pattern:
 		self.__preMatch = None
+		# default TZ (if set, treat log lines without explicit time zone to be in this time zone):
+		self.__default_tz = None
 
 	def _appendTemplate(self, template, ignoreDup=False):
 		name = template.name
@@ -423,7 +426,15 @@ class DateDetector(object):
 		logSys.log(logLevel, " no template.")
 		return (None, None)
 
-	def getTime(self, line, timeMatch=None, default_tz=None):
+	@property
+	def default_tz(self):
+		return self.__default_tz
+
+	@default_tz.setter
+	def default_tz(self, value):
+		self.__default_tz = validateTimeZone(value)
+
+	def getTime(self, line, timeMatch=None):
 		"""Attempts to return the date on a log line using templates.
 
 		This uses the templates' `getDate` method in an attempt to find
@@ -449,7 +460,7 @@ class DateDetector(object):
 		template = timeMatch[1]
 		if template is not None:
 			try:
-				date = template.getDate(line, timeMatch[0], default_tz=default_tz)
+				date = template.getDate(line, timeMatch[0], default_tz=self.__default_tz)
 				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
 						logSys.log(logLevel, "  got time %f for %r using template %s",
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 8d1eb856..a8f99998 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -34,13 +34,12 @@ from .failmanager import FailManagerEmpty, FailManager
 from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
-from .datedetector import DateDetector
+from .datedetector import DateDetector, validateTimeZone
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
 from .utils import Utils
 from ..helpers import getLogger, PREFER_ENC
-from .strptime import validateTimeZone
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -88,6 +87,8 @@ class Filter(JailThread):
 		## Store last time stamp, applicable for multi-line
 		self.__lastTimeText = ""
 		self.__lastDate = None
+		## if set, treat log lines without explicit time zone to be in this time zone
+		self.__logtimezone = None
 		## External command
 		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
@@ -103,8 +104,6 @@ class Filter(JailThread):
 		self.checkAllRegex = False
 		## if true ignores obsolete failures (failure time < now - findTime):
 		self.checkFindTime = True
-		## if set, treat log lines without explicit time zone to be in this time zone
-		self.logtimezone = None
 		## Ticks counter
 		self.ticks = 0
 
@@ -285,6 +284,7 @@ class Filter(JailThread):
 			return
 		else:
 			dd = DateDetector()
+			dd.default_tz = self.__logtimezone
 			if not isinstance(pattern, (list, tuple)):
 				pattern = filter(bool, map(str.strip, re.split('\n+', pattern)))
 			for pattern in pattern:
@@ -316,7 +316,9 @@ class Filter(JailThread):
 	# @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM)
 
 	def setLogTimeZone(self, tz):
-		self.logtimezone = validateTimeZone(tz)
+		validateTimeZone(tz); # avoid setting of wrong value, but hold original
+		self.__logtimezone = tz
+		if self.dateDetector: self.dateDetector.default_tz = self.__logtimezone
 
 	##
 	# Get the log default timezone
@@ -324,7 +326,7 @@ class Filter(JailThread):
 	# @return symbolic timezone (a string)
 
 	def getLogTimeZone(self):
-		return self.logtimezone
+		return self.__logtimezone
 
 	##
 	# Set the maximum retry value.
@@ -640,8 +642,7 @@ class Filter(JailThread):
 			self.__lastDate = date
 		elif timeText:
 
-			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3],
-								  default_tz=self.logtimezone)
+			dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
 
 			if dateTimeMatch is None:
 				logSys.error("findFailure failed to parse timeText: %s", timeText)
@@ -994,8 +995,7 @@ class FileFilter(Filter):
 				if timeMatch:
 					dateTimeMatch = self.dateDetector.getTime(
 						line[timeMatch.start():timeMatch.end()],
-						(timeMatch, template),
-						default_tz=self.logtimezone)
+						(timeMatch, template))
 				else:
 					nextp = container.tell()
 					if nextp > maxp:
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index aff9db92..68ff2f41 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -27,7 +27,7 @@ from .mytime import MyTime
 
 locale_time = LocaleTime()
 timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$')
+FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$')
 
 def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
 	""" Build century regex for last year and the next years (distance).
@@ -84,30 +84,33 @@ def getTimePatternRE():
 def validateTimeZone(tz):
 	"""Validate a timezone.
 
-	For now this accepts only the UTC[+-]hhmm format.
+	For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
 	In the future, it may be extended for named time zones (such as Europe/Paris)
         present on the system, if a suitable tz library is present.
 	"""
+	if tz is None:
+		return None
 	m = FIXED_OFFSET_TZ_RE.match(tz)
 	if m is None:
 		raise ValueError("Unknown or unsupported time zone: %r" % tz)
-	return tz
+	tz = m.group(1)
+	if tz is None or tz == '': # UTC/GMT
+		return 0; # fixed zero offzet
+	return zone2offset(tz, 0)
 
 def zone2offset(tz, dt):
 	"""Return the proper offset, in minutes according to given timezone at a given time.
 
 	Parameters
 	----------
-	tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have
-                               been validated already)
-        dt: datetime instance for offset computation
+	tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have
+		been validated already)
+	dt: datetime instance for offset computation
 	"""
-	if tz == 'UTC':
-		return 0
-	unsigned = int(tz[4:6])*60 + int(tz[6:])
-	if tz[3] == '-':
-		return -unsigned
-	return unsigned
+	if isinstance(tz, basestring):
+		# [+-]1 * (hh*60 + mm)
+		return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+	return tz
 
 def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 16515f1b..6d728b5e 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -91,19 +91,24 @@ class DateDetectorTest(LogCaptureTestCase):
 
 	def testDefaultTimeZone(self):
 		log = "2017-01-23 15:00:00"
-		datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300')
+		dd = self.datedetector
+		dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log)
 		# so in UTC, it was noon
 		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
 				 datetime.datetime(2017, 1, 23, 12, 0, 0))
 
-		datelog, _ = self.datedetector.getTime(log, default_tz='UTC')
+		dd.default_tz='UTC'; datelog, _ = dd.getTime(log)
 		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
 				 datetime.datetime(2017, 1, 23, 15, 0, 0))
+		self.assertEqual(dd.default_tz, 0); # utc == 0
 
-		datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430')
+		dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log)
 		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
 				 datetime.datetime(2017, 1, 23, 19, 30, 0))
 
+		self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
+		dd.default_tz = None
+
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
 		"""

From 9f41d1e3819de407a5e992bd4699566d60c1e941 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 14:55:44 +0200
Subject: [PATCH 414/623] Normalize zone2offset (usable within
 reGroupDictStrptime), tests simplified and extended with more cases (covers
 precedence of input-zone over default, etc.)

---
 fail2ban/server/strptime.py            | 19 +++++++------
 fail2ban/tests/datedetectortestcase.py | 38 ++++++++++++++++----------
 2 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 68ff2f41..56f1c753 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -27,7 +27,7 @@ from .mytime import MyTime
 
 locale_time = LocaleTime()
 timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$')
+FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$')
 
 def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
 	""" Build century regex for last year and the next years (distance).
@@ -108,8 +108,15 @@ def zone2offset(tz, dt):
 	dt: datetime instance for offset computation
 	"""
 	if isinstance(tz, basestring):
-		# [+-]1 * (hh*60 + mm)
-		return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+		if len(tz) <= 3: # short tz (hh only)
+			# [+-]hh --> [+-]hh*60
+			return int(tz)*60
+		if tz[3] != ':':
+			# [+-]hhmm --> [+-]1 * (hh*60 + mm)
+			return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+		else:
+			# [+-]hh:mm --> [+-]1 * (hh*60 + mm)
+			return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6]))
 	return tz
 
 def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
@@ -202,11 +209,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 			if z in ("Z", "UTC", "GMT"):
 				tzoffset = 0
 			else:
-				tzoffset = int(z[1:3]) * 60 # Hours...
-				if len(z)>3:
-					tzoffset += int(z[-2:]) # ...and minutes
-				if z.startswith("-"):
-					tzoffset = -tzoffset
+				tzoffset = zone2offset(z, 0); # currently offset-based only
 		elif key == 'Z':
 			z = val
 			if z in ("UTC", "GMT"):
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 6d728b5e..cbf1ac21 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -90,21 +90,31 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
 
 	def testDefaultTimeZone(self):
-		log = "2017-01-23 15:00:00"
 		dd = self.datedetector
-		dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log)
-		# so in UTC, it was noon
-		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
-				 datetime.datetime(2017, 1, 23, 12, 0, 0))
-
-		dd.default_tz='UTC'; datelog, _ = dd.getTime(log)
-		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
-				 datetime.datetime(2017, 1, 23, 15, 0, 0))
-		self.assertEqual(dd.default_tz, 0); # utc == 0
-
-		dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log)
-		self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
-				 datetime.datetime(2017, 1, 23, 19, 30, 0))
+		dt = datetime.datetime
+		logdt = "2017-01-23 15:00:00"
+		dtUTC = dt(2017, 1, 23, 15, 0)
+		for tz, log, desired in (
+			('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon
+			('UTC',      logdt, dtUTC), # UTC
+			('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
+			('GMT+12',   logdt, dt(2017, 1, 23, 3, 0)),
+			(None,			 logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+			('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
+			('UTC',      logdt+' GMT', dtUTC), # GMT wins
+			('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
+			(None,			 logdt+' GMT', dtUTC), # GMT wins
+			('UTC',      logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins
+			(None,			 logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
+			('UTC',      logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
+			(None,			 logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
+			(None,			 logdt+' Z', dtUTC), # Z wins (UTC)
+		):
+			logSys.debug('== test %r with TZ %r', log, tz)
+			dd.default_tz=tz; datelog, _ = dd.getTime(log)
+			val = dt.utcfromtimestamp(datelog)
+			self.assertEqual(val, desired,
+					 "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
 
 		self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
 		dd.default_tz = None

From 39c4acf6bd0726131bbc3848d514ec66341ce70f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 15:52:14 +0200
Subject: [PATCH 415/623] small amend white-spaces (no functional changes) + a
 bit optimized `zone2offset`

---
 fail2ban/server/strptime.py            | 22 +++++++++++-----------
 fail2ban/tests/datedetectortestcase.py | 10 +++++-----
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 56f1c753..2da27c58 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -107,17 +107,17 @@ def zone2offset(tz, dt):
 		been validated already)
 	dt: datetime instance for offset computation
 	"""
-	if isinstance(tz, basestring):
-		if len(tz) <= 3: # short tz (hh only)
-			# [+-]hh --> [+-]hh*60
-			return int(tz)*60
-		if tz[3] != ':':
-			# [+-]hhmm --> [+-]1 * (hh*60 + mm)
-			return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
-		else:
-			# [+-]hh:mm --> [+-]1 * (hh*60 + mm)
-			return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6]))
-	return tz
+	if isinstance(tz, int):
+		return tz
+	if len(tz) <= 3: # short tz (hh only)
+		# [+-]hh --> [+-]hh*60
+		return int(tz)*60
+	if tz[3] != ':':
+		# [+-]hhmm --> [+-]1 * (hh*60 + mm)
+		return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5]))
+	else:
+		# [+-]hh:mm --> [+-]1 * (hh*60 + mm)
+		return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6]))
 
 def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index cbf1ac21..56970ac5 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -99,16 +99,16 @@ class DateDetectorTest(LogCaptureTestCase):
 			('UTC',      logdt, dtUTC), # UTC
 			('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
 			('GMT+12',   logdt, dt(2017, 1, 23, 3, 0)),
-			(None,			 logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+			(None,       logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
 			('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
 			('UTC',      logdt+' GMT', dtUTC), # GMT wins
 			('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
-			(None,			 logdt+' GMT', dtUTC), # GMT wins
+			(None,       logdt+' GMT', dtUTC), # GMT wins
 			('UTC',      logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins
-			(None,			 logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
+			(None,       logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
 			('UTC',      logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
-			(None,			 logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
-			(None,			 logdt+' Z', dtUTC), # Z wins (UTC)
+			(None,       logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
+			(None,       logdt+' Z', dtUTC), # Z wins (UTC)
 		):
 			logSys.debug('== test %r with TZ %r', log, tz)
 			dd.default_tz=tz; datelog, _ = dd.getTime(log)

From 030f89bf7a7877224e7095fdf34f898354b25f63 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 20:29:34 +0200
Subject: [PATCH 416/623] Implemented zone abbreviations (DST, etc.) and
 abbr+-offset functionality (accept zones like 'CET+0100'), for the list of
 abbreviations see strptime.TZ_STR; Tokens `%z` and `%Z` are more precise now;
 Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations
 and/or offset-based zones;

# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
#       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive),
#       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...'
#       with wrong TZ "error", which is at least not backwards compatible.
#       Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
---
 fail2ban/server/strptime.py            | 114 ++++++++++++++++++++-----
 fail2ban/tests/datedetectortestcase.py |  25 +++++-
 2 files changed, 116 insertions(+), 23 deletions(-)

diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 2da27c58..dbf75d21 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -26,8 +26,9 @@ from _strptime import LocaleTime, TimeRE, _calc_julian_from_U_or_W
 from .mytime import MyTime
 
 locale_time = LocaleTime()
-timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$')
+
+TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?"
+FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,))
 
 def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
 	""" Build century regex for last year and the next years (distance).
@@ -40,10 +41,20 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 		exprset |= set( cent(now[1].year + i) for i in (-1, distance) )
 	return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset)
 
-#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
-#timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
-timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
-timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-]\d{2}(?::?[0-5]\d)?)"
+timeRE = TimeRE()
+
+# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
+#       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), 
+#       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' 
+#       with wrong TZ "error", which is at least not backwards compatible.
+#       Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
+timeRE['Z'] = r"(?P<Z>Z|[A-Z]{3,5})"
+timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)"
+
+# Note: this extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase,
+#       see todo above. Don't use them in default date-patterns (if not anchored, few precise resp. optional).
+timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,)
+timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE)
 
 # Extend build-in TimeRE with some exact patterns
 # exact two-digit patterns:
@@ -82,20 +93,22 @@ def getTimePatternRE():
 
 
 def validateTimeZone(tz):
-	"""Validate a timezone.
+	"""Validate a timezone and convert it to offset if it can (offset-based TZ).
 
-	For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
+	For now this accepts the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
+	Additionally it accepts all zone abbreviations mentioned below in TZ_STR.
+	Note that currently this zone abbreviations are offset-based and used fixed
+	offset without automatically DST-switch (if CET used then no automatically CEST-switch).
+	
 	In the future, it may be extended for named time zones (such as Europe/Paris)
-        present on the system, if a suitable tz library is present.
+	present on the system, if a suitable tz library is present (pytz).
 	"""
 	if tz is None:
 		return None
 	m = FIXED_OFFSET_TZ_RE.match(tz)
 	if m is None:
 		raise ValueError("Unknown or unsupported time zone: %r" % tz)
-	tz = m.group(1)
-	if tz is None or tz == '': # UTC/GMT
-		return 0; # fixed zero offzet
+	tz = m.groups()
 	return zone2offset(tz, 0)
 
 def zone2offset(tz, dt):
@@ -103,21 +116,29 @@ def zone2offset(tz, dt):
 
 	Parameters
 	----------
-	tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have
-		been validated already)
-	dt: datetime instance for offset computation
+	tz: symbolic timezone or offset (for now only TZA?([+-]hh:?mm?)? is supported,
+		as value are accepted:
+		  int offset;
+		  string in form like 'CET+0100' or 'UTC' or '-0400';
+		  tuple (or list) in form (zone name, zone offset);
+	dt: datetime instance for offset computation (currently unused)
 	"""
 	if isinstance(tz, int):
 		return tz
-	if len(tz) <= 3: # short tz (hh only)
+	if isinstance(tz, basestring):
+		return validateTimeZone(tz)
+	tz, tzo = tz
+	if tzo is None or tzo == '': # without offset
+		return TZ_ABBR_OFFS[tz]
+	if len(tzo) <= 3: # short tzo (hh only)
 		# [+-]hh --> [+-]hh*60
-		return int(tz)*60
-	if tz[3] != ':':
+		return TZ_ABBR_OFFS[tz] + int(tzo)*60
+	if tzo[3] != ':':
 		# [+-]hhmm --> [+-]1 * (hh*60 + mm)
-		return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5]))
+		return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[3:5]))
 	else:
 		# [+-]hh:mm --> [+-]1 * (hh*60 + mm)
-		return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6]))
+		return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[4:6]))
 
 def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""Return time from dictionary of strptime fields
@@ -275,3 +296,56 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	if msec: # pragma: no cover - currently unused
 		tm += fraction/1000000.0
 	return tm
+
+
+TZ_ABBR_OFFS = {'':0, None:0}
+TZ_STR = '''
+	-12 Y
+	-11 X NUT SST
+	-10 W CKT HAST HST TAHT TKT
+	-9 V AKST GAMT GIT HADT HNY
+	-8 U AKDT CIST HAY HNP PST PT
+	-7 T HAP HNR MST PDT
+	-6 S CST EAST GALT HAR HNC MDT
+	-5 R CDT COT EASST ECT EST ET HAC HNE PET
+	-4 Q AST BOT CLT COST EDT FKT GYT HAE HNA PYT
+	-3 P ADT ART BRT CLST FKST GFT HAA PMST PYST SRT UYT WGT
+	-2 O BRST FNT PMDT UYST WGST
+	-1 N AZOT CVT EGT
+	0 Z EGST GMT UTC WET WT
+	1 A CET DFT WAT WEDT WEST
+	2 B CAT CEDT CEST EET SAST WAST
+	3 C EAT EEDT EEST IDT MSK
+	4 D AMT AZT GET GST KUYT MSD MUT RET SAMT SCT
+	5 E AMST AQTT AZST HMT MAWT MVT PKT TFT TJT TMT UZT YEKT
+	6 F ALMT BIOT BTT IOT KGT NOVT OMST YEKST
+	7 G CXT DAVT HOVT ICT KRAT NOVST OMSST THA WIB
+	8 H ACT AWST BDT BNT CAST HKT IRKT KRAST MYT PHT SGT ULAT WITA WST
+	9 I AWDT IRKST JST KST PWT TLT WDT WIT YAKT
+	10 K AEST ChST PGT VLAT YAKST YAPT
+	11 L AEDT LHDT MAGT NCT PONT SBT VLAST VUT
+	12 M ANAST ANAT FJT GILT MAGST MHT NZST PETST PETT TVT WFT
+	13 FJST NZDT
+	11.5 NFT
+	10.5 ACDT LHST
+	9.5 ACST
+	6.5 CCT MMT
+	5.75 NPT
+	5.5 SLT
+	4.5 AFT IRDT
+	3.5 IRST
+	-2.5 HAT NDT
+	-3.5 HNT NST NT
+	-4.5 HLV VET
+	-9.5 MART MIT
+'''
+
+def _init_TZ_ABBR():
+	"""Initialized TZ_ABBR_OFFS dictionary (TZ -> offset in minutes)"""
+	for tzline in map(str.split, TZ_STR.split('\n')):
+		if not len(tzline): continue
+		tzoffset = int(float(tzline[0]) * 60)
+		for tz in tzline[1:]:
+			TZ_ABBR_OFFS[tz] = tzoffset 
+
+_init_TZ_ABBR()
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 56970ac5..02facf30 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -90,16 +90,32 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
 
 	def testDefaultTimeZone(self):
-		dd = self.datedetector
+		# use special date-pattern (with %Exz), because %z currently does not supported 
+		# zone abbreviations except Z|UTC|GMT.
+		dd = DateDetector()
+		dd.appendTemplate('^%ExY-%Exm-%Exd %H:%M:%S(?: ?%Exz)?')
 		dt = datetime.datetime
 		logdt = "2017-01-23 15:00:00"
 		dtUTC = dt(2017, 1, 23, 15, 0)
 		for tz, log, desired in (
+			# no TZ in input-string:
 			('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon
 			('UTC',      logdt, dtUTC), # UTC
 			('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
 			('GMT+12',   logdt, dt(2017, 1, 23, 3, 0)),
 			(None,       logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+			# CET:
+			('CET',      logdt, dt(2017, 1, 23, 14, 0)),
+			('+0100',    logdt, dt(2017, 1, 23, 14, 0)),
+			('CEST-01',  logdt, dt(2017, 1, 23, 14, 0)),
+			# CEST:
+			('CEST',     logdt, dt(2017, 1, 23, 13, 0)),
+			('+0200',    logdt, dt(2017, 1, 23, 13, 0)),
+			('CET+01',   logdt, dt(2017, 1, 23, 13, 0)),
+			('CET+0100', logdt, dt(2017, 1, 23, 13, 0)),
+			# check offset in minutes:
+			('CET+0130', logdt, dt(2017, 1, 23, 12, 30)),
+			# TZ in input-string have precedence:
 			('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
 			('UTC',      logdt+' GMT', dtUTC), # GMT wins
 			('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
@@ -108,13 +124,16 @@ class DateDetectorTest(LogCaptureTestCase):
 			(None,       logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
 			('UTC',      logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
 			(None,       logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
-			(None,       logdt+' Z', dtUTC), # Z wins (UTC)
+			('UTC+0300', logdt+' Z', dtUTC), # Z wins (UTC)
+			('GMT+12',   logdt+' CET',  dt(2017, 1, 23, 14, 0)), # CET wins
+			('GMT+12',   logdt+' CEST', dt(2017, 1, 23, 13, 0)), # CEST wins
+			('GMT+12',   logdt+' CET+0130', dt(2017, 1, 23, 12, 30)), # CET+0130 wins
 		):
 			logSys.debug('== test %r with TZ %r', log, tz)
 			dd.default_tz=tz; datelog, _ = dd.getTime(log)
 			val = dt.utcfromtimestamp(datelog)
 			self.assertEqual(val, desired,
-					 "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
+					 "wrong offset %r != %r by %r with default TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
 
 		self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
 		dd.default_tz = None

From 12259bb3c7aac2e33b9cc8f601c0e556b5401bd5 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Fri, 9 Jun 2017 20:39:03 +0200
Subject: [PATCH 417/623] man and ChangeLog for logtimezone

---
 ChangeLog       |  2 +-
 man/jail.conf.5 | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index cd4776ee..7c99da91 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -98,7 +98,7 @@ TODO: implementing of options resp. other tasks from PR #1346
   e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
   the actions having `actionflush` do not execute `actionunban` for each single ticket
 * add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
-
+* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
 -----------
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 5a75369c..a43346e1 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -177,6 +177,25 @@ Ensure syslog or the program that generates the log file isn't configured to com
 .TP
 .B logencoding
 encoding of log files used for decoding. Default value of "auto" uses current system locale.
+.TP
+.B logtimezone
+Force the time zone for log lines that don't have one.
+
+If this option is not specified, log lines from which no explicit time zone has been found are interpreted by fail2ban in its own system time zone, and that may turn to be inappropriate. While the best practice is to configure the monitored applications to include explicit offsets, this option is meant to handle cases where that is not possible.
+
+The supported time zones in this option are those with fixed offset: Z, UTC[+-]hhmm (you can also use GMT as an alias to UTC).
+
+This option has no effect on log lines on which an explicit time zone has been found.
+Examples:
+
+.RS
+.nf
+        logtimezone = UTC
+        logtimezone = UTC+0200
+        logtimezone = GMT-0100
+.fi
+.RE
+
 .TP
 .B banaction
 banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails.

From 23c2d05250db5da1cf111da96115fce0e41c9ea3 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 9 Jun 2017 20:51:28 +0200
Subject: [PATCH 418/623] Update changelog (new enhancements from gh-1792)

---
 ChangeLog | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 7c99da91..a2eac62b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -88,17 +88,32 @@ TODO: implementing of options resp. other tasks from PR #1346
     the parsing of log-entries contain new-line chars (as single entry);
   - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` 
     could be used in regex to enable it;
-* implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742):
+* Implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742):
   - new action parameter `actionstart_on_demand` (bool) can be set to prevent/allow starting action
     on demand (default retrieved automatically, if some conditional parameter `param?family=...` 
     presents in action properties), see `action.d/pf.conf` for example;
   - additionally `actionstop` will be executed only for families previously executing `actionstart`
     (starting on demand only)
-* introduced new command `actionflush`: executed in order to flush all bans at once 
+* Introduced new command `actionflush`: executed in order to flush all bans at once 
   e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
   the actions having `actionflush` do not execute `actionunban` for each single ticket
-* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
-* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
+* Add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
+* Add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
+* Implemented zone abbreviations (like CET, CEST, etc.) and abbr+-offset functionality (accept zones
+  like 'CET+0100'), for the list of abbreviations see strptime.TZ_STR;
+* Tokens `%z` and `%Z` are changed (more precise now);
+* Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations and/or offset-based
+  zones (implemented as enhancement using custom `datepattern`, because may be too dangerous for default
+  patterns and tokens like `%z`);
+  Note: the extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase.
+        Don't use them in default date-patterns (if not anchored, few precise resp. optional).
+        Because python currently does not support mixing of case-sensitive with case-insensitive matching,
+	the TZ (in uppercase) cannot be combined with `%a`/`%b` etc (that are currently case-insensitive),
+	to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' with
+	wrong TZ "error".
+        Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone 
+	abbreviations.
+
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
 -----------

From 6f3d425c4d4d8d86faa67a3e058e1426c74a7680 Mon Sep 17 00:00:00 2001
From: jeaye <contact@jeaye.com>
Date: Mon, 12 Jun 2017 10:53:28 -0700
Subject: [PATCH 419/623] Update postfix filters and tests

---
 ChangeLog                         | 6 ++++--
 THANKS                            | 1 +
 config/filter.d/postfix.conf      | 6 +++---
 fail2ban/tests/files/logs/postfix | 9 +++++++++
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 33a0c926..54d48140 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -22,10 +22,12 @@ TODO: implementing of options resp. other tasks from PR #1346
     (by using tag `<F-MLFID>` instead of buffering with `maxlines`);
   - optional parameter `mode` rewritten: normal (default), ddos, extra or aggressive (combines all),
     see sshd for regex details)
-* filter.d/sendmail-reject.conf:
+* `filter.d/sendmail-reject.conf`:
   - rewritten using `prefregex` and used MLFID-related multi-line parsing;
   - optional parameter `mode` introduced: normal (default), extra or aggressive
-* filter.d/haproxy-http-auth: do not mistake client port for part of an IPv6 address (gh-1745)
+* `filter.d/haproxy-http-auth`: do not mistake client port for part of an IPv6 address (gh-1745)
+* `filter.d/postfix-sasl.conf`
+    - updated to latest postfix formats
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`
diff --git a/THANKS b/THANKS
index 8cc8f7bd..7861ceb5 100644
--- a/THANKS
+++ b/THANKS
@@ -61,6 +61,7 @@ John Thoe
 Jacques Lav!gnotte
 Johannes Weberhofer
 Jason H Martin
+Jeaye Wilkerson
 Jisoo Park
 Joel M Snyder
 Jonathan Kamens
diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index 005ed585..1dbc23c0 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -15,12 +15,12 @@ _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
 prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) <F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^RCPT from \S+\[<HOST>\]: 554 5\.7\.1 
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 (<\S+>)?: Helo command rejected: Host not found; from=<\S*> to=<\S*> proto=\S+ helo=.*$
             ^EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
             ^VRFY from \S+\[<HOST>\]: 550 5\.1\.1 
             ^RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^after \S+ from [^[]*\[<HOST>\]:?$
+            ^after \S+ from [^[]*\[<HOST>\]:?
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix
index 78f72287..c38ba311 100644
--- a/fail2ban/tests/files/logs/postfix
+++ b/fail2ban/tests/files/logs/postfix
@@ -35,3 +35,12 @@ Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s27
 
 # failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" }
 Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: NOQUEUE: reject: EHLO from astra4139.startdedicated.de[62.138.2.143]: 504 5.5.2 <User>: Helo command rejected: need fully-qualified hostname; proto=SMTP helo=<User>
+
+# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" }
+Jun 12 08:58:35 xxx postfix/smtpd[27296]: NOQUEUE: reject: RCPT from unknown[1.2.3.4]: 450 4.7.1 Client host rejected: cannot find your reverse hostname, [2.3.4.5]; from=<meow@kitty.com> to=<kitty@meow.com> proto=ESMTP helo=<kitty.com>
+
+# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" }
+Jun 12 08:58:35 xxx postfix/smtpd[2931]: NOQUEUE: reject: RCPT from unknown[1.2.3.4]: 450 4.7.1 <kitty.com>: Helo command rejected: Host not found; from=<meow@kitty.com> to=<kitty@meow.com> proto=SMTP helo=<kitty.com>
+
+# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" }
+Jun 12 08:58:35 xxx postfix/smtpd[13533]: improper command pipelining after AUTH from unknown[1.2.3.4]: QUIT

From 2b358bc1a4c985c07120d8b4bc7e025339926bdc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 11:16:19 +0200
Subject: [PATCH 420/623] filter.d/apache-overflows.conf: rewritten without
 end-anchor ($), because apache-log could contain very long URLs (and/or
 referrer), the parsing of it anchored way may be very vulnerable (at least as
 regards the system resources, see gh-1790).

---
 config/filter.d/apache-overflows.conf | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf
index 74e44b8e..02a2ef20 100644
--- a/config/filter.d/apache-overflows.conf
+++ b/config/filter.d/apache-overflows.conf
@@ -8,11 +8,15 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)(, referer: \S+)?$
+failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
 
 ignoreregex =
 
 # DEV Notes:
+#
+# [sebres] Because this apache-log could contain very long URLs (and/or referrer), 
+#          the parsing of it anchored way may be very vulnerable (at least as regards 
+#          the system resources, see gh-1790). Thus rewritten without end-anchor ($).
 # 
 # fgrep -r 'URI too long' httpd-2.*
 #   httpd-2.2.25/server/protocol.c:                          "request failed: URI too long (longer than %d)", r->server->limit_req_line);

From e1234a5249d7d9ca604970d80c54353a86b60aba Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 11:40:07 +0200
Subject: [PATCH 421/623] ChangeLog update

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5f472f81..e99bab04 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -17,6 +17,9 @@ releases.
 ### Fixes
 * Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991.
   Partially back-ported from v.0.10.
+* filter.d/apache-overflows.conf:
+    - Fixes resources greedy expression (see gh-1790);
+    - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
 
 ### New Features
 

From fd32e908e31add6f5eed677ce2c2f3e77e50ee49 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 18:28:37 +0200
Subject: [PATCH 422/623] fixes restoring of tickets from database for jails
 with persistent ban (if `bantime = -1`)

---
 fail2ban/server/database.py        |  2 +-
 fail2ban/tests/databasetestcase.py | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 77786f57..f4f9b6c2 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -593,7 +593,7 @@ class Fail2BanDb(object):
 		if ip is not None:
 			query += " AND ip=?"
 			queryArgs.append(ip)
-		if forbantime is not None:
+		if forbantime not in (None, -1): # not specified or persistent (all)
 			query += " AND timeofban > ?"
 			queryArgs.append(fromtime - forbantime)
 		if ip is None:
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index b1c68a8f..cbfc1517 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -358,6 +358,19 @@ class DatabaseTest(LogCaptureTestCase):
 		self.assertEqual(len(tickets), 2)
 		ticket = self.db.getCurrentBans(jail=None, ip="127.0.0.1");
 		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		
+		# positive case (1 ticket not yet expired):
+		tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15,
+			fromtime=MyTime.time())
+		self.assertEqual(len(tickets), 1)
+		# negative case (all are expired in 1year):
+		tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15,
+			fromtime=MyTime.time() + MyTime.str2seconds("1year"))
+		self.assertEqual(len(tickets), 0)
+		# persistent bantime (-1), so never expired:
+		tickets = self.db.getCurrentBans(jail=self.jail, forbantime=-1,
+			fromtime=MyTime.time() + MyTime.str2seconds("1year"))
+		self.assertEqual(len(tickets), 2)
 
 	def testActionWithDB(self):
 		# test action together with database functionality

From efeca8fdeb7470ba5605762a8b9a97fa56943204 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 19 Jun 2017 16:25:46 +0200
Subject: [PATCH 423/623] postfix.conf: removes unneeded end-anchoring like
 `.*$`, etc.

also removes several dynamic content at end, which are of no avail there.
Additionally normalizes optional part (mail-ID) after reason number.
---
 config/filter.d/postfix.conf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index 1dbc23c0..1cf54f14 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -15,11 +15,11 @@ _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
 prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) <F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^RCPT from \S+\[<HOST>\]: 554 5\.7\.1 
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 (<\S+>)?: Helo command rejected: Host not found; from=<\S*> to=<\S*> proto=\S+ helo=.*$
-            ^EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname\b
+            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b
+            ^EHLO from \S+\[<HOST>\]: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
             ^VRFY from \S+\[<HOST>\]: 550 5\.1\.1 
-            ^RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^RCPT from \S+\[<HOST>\]: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b
             ^after \S+ from [^[]*\[<HOST>\]:?
 
 ignoreregex = 

From 691c080dc747ba159bb3e68b1a7725a4ab2f23cc Mon Sep 17 00:00:00 2001
From: Johannes Weberhofer <web@c-web3.vie.weberhofer.at>
Date: Fri, 19 May 2017 15:23:58 +0200
Subject: [PATCH 424/623] Added roundcube authentication filter, new jail and
 log-examples

---
 config/filter.d/roundcube.conf      | 21 +++++++++++++++++++++
 config/jail.conf                    |  4 ++++
 fail2ban/tests/files/logs/roundcube |  4 ++++
 3 files changed, 29 insertions(+)
 create mode 100644 config/filter.d/roundcube.conf
 create mode 100644 fail2ban/tests/files/logs/roundcube

diff --git a/config/filter.d/roundcube.conf b/config/filter.d/roundcube.conf
new file mode 100644
index 00000000..9665883b
--- /dev/null
+++ b/config/filter.d/roundcube.conf
@@ -0,0 +1,21 @@
+# Fail2Ban configuration file for roundcube web server authentication failures
+#
+# This filter needs "$config['log_driver']" set to "syslog" in the roundcube configuration
+#
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+failregex = ^%(__prefix_line)sroundcube\[(\d*)\]: <\S* IMAP Error: Login failed for (\S*) from <HOST>\..*$
+
+ignoreregex =
+
+[Init]
+
+backend  = systemd
+
+journalmatch = SYSLOG_IDENTIFIER=roundcube
+
diff --git a/config/jail.conf b/config/jail.conf
index 135c9a2b..bd9ba876 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -379,6 +379,10 @@ logpath = %(lighttpd_error_log)s
 # Webmail and groupware servers
 #
 
+[roundcube]
+port    = http,https
+backend = %(syslog_backend)s
+
 [roundcube-auth]
 
 port     = http,https
diff --git a/fail2ban/tests/files/logs/roundcube b/fail2ban/tests/files/logs/roundcube
new file mode 100644
index 00000000..b491fa2b
--- /dev/null
+++ b/fail2ban/tests/files/logs/roundcube
@@ -0,0 +1,4 @@
+May 19 06:07:48 server roundcube[21296]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
+May 19 06:11:37 server roundcube[22926]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
+May 19 06:13:18 server roundcube[21528]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
+May 19 06:36:53 server roundcube[27572]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)

From d3ae70beb6f73f289843157307f079678c156457 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 19 Jun 2017 18:05:29 +0200
Subject: [PATCH 425/623] filter.d/roundcube-auth.conf: Use the same
 filter-file and jail also when logging errors to journal instead to a local
 file. Additionally fixes more complex injections on username.

---
 config/filter.d/roundcube-auth.conf      |  6 ++++--
 config/filter.d/roundcube.conf           | 21 ---------------------
 config/jail.conf                         |  6 ++----
 fail2ban/tests/files/logs/roundcube      |  4 ----
 fail2ban/tests/files/logs/roundcube-auth | 19 +++++++++++++++----
 5 files changed, 21 insertions(+), 35 deletions(-)
 delete mode 100644 config/filter.d/roundcube.conf
 delete mode 100644 fail2ban/tests/files/logs/roundcube

diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf
index 886cf2d6..bab62651 100644
--- a/config/filter.d/roundcube-auth.conf
+++ b/config/filter.d/roundcube-auth.conf
@@ -13,8 +13,10 @@ before = common.conf
 
 [Definition]
 
-failregex = ^\s*(\[\])?(%(__hostname)s\s*(roundcube:)?\s*(<[\w]+>)? IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. .* in .*?/rcube_imap\.php on line \d+ \(\S+ \S+\))?$
-            ^\[\]:\s*(<[\w]+>)? Failed login for [\w\-\.\+]+(@[\w\-\.\+]+\.[a-zA-Z]{2,6})? from <HOST> in session \w+( \(error: \d\))?$
+prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+>)? IMAP Error)?: <F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
+            ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
 
 ignoreregex = 
 # DEV Notes:
diff --git a/config/filter.d/roundcube.conf b/config/filter.d/roundcube.conf
deleted file mode 100644
index 9665883b..00000000
--- a/config/filter.d/roundcube.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Fail2Ban configuration file for roundcube web server authentication failures
-#
-# This filter needs "$config['log_driver']" set to "syslog" in the roundcube configuration
-#
-
-[INCLUDES]
-
-before = common.conf
-
-[Definition]
-
-failregex = ^%(__prefix_line)sroundcube\[(\d*)\]: <\S* IMAP Error: Login failed for (\S*) from <HOST>\..*$
-
-ignoreregex =
-
-[Init]
-
-backend  = systemd
-
-journalmatch = SYSLOG_IDENTIFIER=roundcube
-
diff --git a/config/jail.conf b/config/jail.conf
index bd9ba876..21bc898e 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -379,14 +379,12 @@ logpath = %(lighttpd_error_log)s
 # Webmail and groupware servers
 #
 
-[roundcube]
-port    = http,https
-backend = %(syslog_backend)s
-
 [roundcube-auth]
 
 port     = http,https
 logpath  = %(roundcube_errors_log)s
+# Use following line in your jail.local if roundcube logs to journal.
+#backend = %(syslog_backend)s
 
 
 [openwebmail]
diff --git a/fail2ban/tests/files/logs/roundcube b/fail2ban/tests/files/logs/roundcube
deleted file mode 100644
index b491fa2b..00000000
--- a/fail2ban/tests/files/logs/roundcube
+++ /dev/null
@@ -1,4 +0,0 @@
-May 19 06:07:48 server roundcube[21296]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
-May 19 06:11:37 server roundcube[22926]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
-May 19 06:13:18 server roundcube[21528]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
-May 19 06:36:53 server roundcube[27572]: <crk9n97i> IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)
diff --git a/fail2ban/tests/files/logs/roundcube-auth b/fail2ban/tests/files/logs/roundcube-auth
index 26868c3e..8c491b38 100644
--- a/fail2ban/tests/files/logs/roundcube-auth
+++ b/fail2ban/tests/files/logs/roundcube-auth
@@ -8,19 +8,27 @@ Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.
 # Made up to attempts to inject a DoS on the server. Assume the user can manipulate the IMAP error response
 #
 # user = admin from 127.0.0.1
-# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" }
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 1" }
 Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
+# user = admin from 127.0.0.1.
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 1 (with dot)" }
+Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1. from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
+#
 #
 # IMAP server logs user=${username}
-# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" }
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 2" }
 Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
 #
+# IMAP server logs user=${username}
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 2 (with dot)" }
+Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1. from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
+#
 # Old roundcube version - no IMAP response
-# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" }
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 3" }
 Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4
 #
 # user = admin from 127.0.0.1 in 
-# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" }
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 4" }
 Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 in from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
 
 # Roundcube 1.0.5 CentOS 6 (/var/log/roundcubemail/errors)
@@ -40,3 +48,6 @@ Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 12
 # Roundcube 1.1.1 (/var/log/roundcubemail/userlogins)
 # failJSON: { "time": "2015-05-10T19:02:52", "match": true , "host": "1.2.3.4" }
 [10-May-2015 13:02:52 -0400]: <4z506z6r> Failed login for admin@example.com from 1.2.3.4 in session 4z506z6rvddstv6k7jz08hxo27 (error: 0)
+
+# failJSON: { "time": "2005-05-19T06:07:48", "match": true , "host": "192.0.2.1", "desc": "Roundcube logged to journald instead to a local file."}
+May 19 06:07:48 server roundcube[21296]: <crk9n97i> IMAP Error: Login failed for test from 192.0.2.1. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)

From 9b0f39a17d32b2f78738c8500123cf01b56b550b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 19 Jun 2017 18:07:48 +0200
Subject: [PATCH 426/623] ChangeLog updated

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c53989f6..cf4411cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,6 +28,9 @@ TODO: implementing of options resp. other tasks from PR #1346
 * `filter.d/haproxy-http-auth`: do not mistake client port for part of an IPv6 address (gh-1745)
 * `filter.d/postfix-sasl.conf`
     - updated to latest postfix formats
+* `filter.d/roundcube-auth.conf`:
+    - fixed regex when logging authentication errors to journal instead to a local file (gh-1159);
+    - additionally fixed more complex injections on username (e. g. using dot after fake host).
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`

From 37ca4f17c2b7d34f9d99879f438eaced928c6b4e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 26 Jun 2017 11:24:10 +0200
Subject: [PATCH 427/623] filter.d/roundcube-auth.conf: added missing entry
 `journalmatch` from original gh-1783.

---
 config/filter.d/roundcube-auth.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf
index bab62651..1d0007da 100644
--- a/config/filter.d/roundcube-auth.conf
+++ b/config/filter.d/roundcube-auth.conf
@@ -19,6 +19,9 @@ failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(\
             ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
 
 ignoreregex = 
+
+journalmatch = SYSLOG_IDENTIFIER=roundcube
+
 # DEV Notes:
 #
 # Source: https://github.com/roundcube/roundcubemail/blob/master/program/lib/Roundcube/rcube_imap.php#L180

From 6110ba9cc34e212d0ff6ac5be4b3d5bebadef794 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 30 Jun 2017 18:00:01 +0200
Subject: [PATCH 428/623] filter.d/proftpd.conf: added option `journalmatch`
 for systemd backend (closes gh-1613)

---
 config/filter.d/proftpd.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf
index 4bc0ba01..303be5e5 100644
--- a/config/filter.d/proftpd.conf
+++ b/config/filter.d/proftpd.conf
@@ -23,5 +23,8 @@ failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .*: no
 
 ignoreregex = 
 
+[Init]
+journalmatch = _SYSTEMD_UNIT=proftpd.service
+
 # Author: Yaroslav Halchenko
 #         Daniel Black - hardening of regex

From f27e053592767ef1fd37da0cf242ee49c6bbebcc Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Sat, 1 Jul 2017 17:10:53 +0200
Subject: [PATCH 429/623] Update bsd-ipfw.conf

increased starting rule number (lowest_rule_num = 111)
---
 config/action.d/bsd-ipfw.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf
index 65d4294a..65a5e39b 100644
--- a/config/action.d/bsd-ipfw.conf
+++ b/config/action.d/bsd-ipfw.conf
@@ -86,6 +86,6 @@ blocktype = unreach port
 # Notes:   When fail2ban starts with action and there is no rule for the given table yet
 #          then fail2ban will start looking for an empty slot starting with this rule number.
 # Values:  NUM
-lowest_rule_num = 1
+lowest_rule_num = 111
 
 

From 9f55ed86df19d990890c0103f038a147cb7145b9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 3 Jul 2017 12:38:21 +0200
Subject: [PATCH 430/623] fixed testCymruInfoNxdomain (since cymru does not
 provide ASN mapping info for "10.0.0.0" anymore)

---
 fail2ban/tests/banmanagertestcase.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 8c58994c..ec1e5686 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -106,13 +106,12 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 						   "country": ["nxdomain"],
 						   "rir": ["nxdomain"]})
 
-		# even for private IPs ASNs defined
 		# Since it outputs for all active tickets we would get previous results
 		# and new ones
-		ticket = BanTicket("10.0.0.0", 1167606000.0)
+		ticket = BanTicket("8.0.0.0", 1167606000.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
 		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 		assert_dict_equal(cymru_info,
-						  {"asn": ["nxdomain", "4565",],
-						   "country": ["nxdomain", "unknown"],
-						   "rir": ["nxdomain", "other"]})
+						  {"asn": ["nxdomain", "3356",],
+						   "country": ["nxdomain", "US"],
+						   "rir": ["nxdomain", "arin"]})

From a1d0633e69d13cd2beaac27f039d8bb9d0fdc47b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 3 Jul 2017 12:57:28 +0200
Subject: [PATCH 431/623] filter.d/asterisk.conf - fixed failregex AMI Asterisk
 authentification failed (see gh-1302): - optional space between NOTICE and
 pid; - optional part "Host " before IP-address;

---
 ChangeLog                          | 1 +
 config/filter.d/asterisk.conf      | 6 +++---
 fail2ban/tests/files/logs/asterisk | 3 +++
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3e9ef653..99bcb4b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -23,6 +23,7 @@ releases.
 * filter.d/apache-overflows.conf:
     - Fixes resources greedy expression (see gh-1790);
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
+* filter.d/asterisk.conf - fixed failregex AMI Asterisk authentification failed (see gh-1302)
 
 ### New Features
 
diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 79bd1ff2..97178002 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -11,7 +11,7 @@ before = common.conf
 
 _daemon = asterisk
 
-__pid_re = (?:\[\d+\])
+__pid_re = (?:\s*\[\d+\])
 
 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 
@@ -20,9 +20,9 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\
 
 failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
             ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
+            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed to authenticate
             ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
             ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 13808592..f3ece02e 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -84,3 +84,6 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # Failed authentication with pjsip on Asterisk 13+
 # failJSON: { "time": "2016-06-08T23:40:26", "match": true , "host": "2.3.4.5" }
 [2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found
+
+# failJSON: { "time": "2016-06-09T00:01:02", "match": true , "host": "192.0.2.1" }
+[2016-06-09 00:01:02] NOTICE [22382] manager.c: 192.0.2.1 failed to authenticate as 'admin'
\ No newline at end of file

From 57ea38c342b2caa17b61a8cd17f142c218fd0742 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 5 Jul 2017 19:57:30 +0200
Subject: [PATCH 432/623] Update paths-debian.conf

Fixed mail.log path since in the default rsyslog configuration of debians the `mail.warn` is commented now (see `/etc/rsyslog.d/50-default.conf`: `#mail.warn -/var/log/mail.warn`).
Closes gh-1687
---
 config/paths-debian.conf | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/paths-debian.conf b/config/paths-debian.conf
index 4c27dac8..1ca4d242 100644
--- a/config/paths-debian.conf
+++ b/config/paths-debian.conf
@@ -11,7 +11,9 @@ after  = paths-overrides.local
 
 syslog_mail = /var/log/mail.log
 
-syslog_mail_warn = /var/log/mail.warn
+# control the `mail.warn` setting, see `/etc/rsyslog.d/50-default.conf` (if commented `mail.*` wins).
+# syslog_mail_warn = /var/log/mail.warn
+syslog_mail_warn = %(syslog_mail)s
 
 syslog_authpriv = /var/log/auth.log
 

From d32a3913cfb08bd2cebe041204bfe14f060da0c3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 10 Jul 2017 15:38:24 +0200
Subject: [PATCH 433/623] postfix postscreen (resp. other RBL's compatibility
 fix) / gh-1764

---
 config/filter.d/postfix.conf      | 15 ++++++++-------
 fail2ban/tests/files/logs/postfix |  3 +++
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index 1cf54f14..e6a3e5c2 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -10,16 +10,17 @@ before = common.conf
 
 [Definition]
 
-_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+_daemon = postfix(-\w+)?/\w+(?:/smtp[ds])?
+_port = (?::\d+)?
 
 prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) <F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^RCPT from \S+\[<HOST>\]: 554 5\.7\.1 
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname\b
-            ^RCPT from \S+\[<HOST>\]: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b
-            ^EHLO from \S+\[<HOST>\]: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
-            ^VRFY from \S+\[<HOST>\]: 550 5\.1\.1 
-            ^RCPT from \S+\[<HOST>\]: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b
+failregex = ^RCPT from [^[]*\[<HOST>\]<_port>: 55[04] 5\.7\.1\s
+            ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname\b
+            ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b
+            ^EHLO from [^[]*\[<HOST>\]<_port>: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
+            ^VRFY from [^[]*\[<HOST>\]<_port>: 550 5\.1\.1\s
+            ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b
             ^after \S+ from [^[]*\[<HOST>\]:?
 
 ignoreregex = 
diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix
index c38ba311..7f24cdd0 100644
--- a/fail2ban/tests/files/logs/postfix
+++ b/fail2ban/tests/files/logs/postfix
@@ -44,3 +44,6 @@ Jun 12 08:58:35 xxx postfix/smtpd[2931]: NOQUEUE: reject: RCPT from unknown[1.2.
 
 # failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" }
 Jun 12 08:58:35 xxx postfix/smtpd[13533]: improper command pipelining after AUTH from unknown[1.2.3.4]: QUIT
+
+# failJSON: { "time": "2005-05-05T15:51:11", "match": true , "host": "216.245.194.173", "desc": "postfix postscreen / gh-1764" }
+May  5 15:51:11 xxx postfix/postscreen[1148]: NOQUEUE: reject: RCPT from [216.245.194.173]:60591: 550 5.7.1 Service unavailable; client [216.245.194.173] blocked using rbl.example.com; from=<spammer@example.com>, to=<goodguy@example.com>, proto=ESMTP, helo=<badguy.example.com>

From 36d42d7f0bb795a271990a7aceb5de783d136467 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 10 Jul 2017 19:57:02 +0200
Subject: [PATCH 434/623] SampleRegexsFactory: introduce opportunity to supply
 multiple options combinations (check lines using filters with several
 options), see for example filter sshd.conf

---
 fail2ban/tests/files/logs/sshd    |   6 +-
 fail2ban/tests/samplestestcase.py | 195 +++++++++++++++++-------------
 2 files changed, 113 insertions(+), 88 deletions(-)

diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index fb3defea..b9559359 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -1,3 +1,5 @@
+# filterOptions: [{}, {"mode": "aggressive"}]
+
 #1
 # failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.030.0.6" }
 Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6
@@ -189,7 +191,7 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S
 # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" }
 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217  user=root
 
-# filterOptions: {"mode": "ddos"}
+# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}]
 
 # http://forums.powervps.com/showthread.php?t=1667
 # failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" }
@@ -218,7 +220,7 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res
 Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth]
 
 
-# filterOptions: {"mode": "extra"}
+# filterOptions: [{"mode": "extra"}, {"mode": "aggressive"}]
 
 # several other cases from gh-864:
 # failJSON: { "time": "2004-11-25T01:34:12", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" }
diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py
index 121c1c5c..5f0a447a 100644
--- a/fail2ban/tests/samplestestcase.py
+++ b/fail2ban/tests/samplestestcase.py
@@ -49,7 +49,8 @@ class FilterSamplesRegex(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		super(FilterSamplesRegex, self).setUp()
-		self.filter = None
+		self._filters = dict()
+		self._filterTests = None
 		setUpMyTime()
 
 	def tearDown(self):
@@ -79,14 +80,20 @@ class FilterSamplesRegex(unittest.TestCase):
 			RE_WRONG_GREED.search('non-greedy .+? test' + RE_HOST + ' test vary catch-all .* anchored$'))
 
 
-	def _readFilter(self, name, basedir, opts=None):
-		self.filter = Filter(None)
-		self.filter.returnRawHost = True
-		self.filter.checkAllRegex = True
-		self.filter.checkFindTime = False
-		self.filter.active = True
+	def _readFilter(self, fltName, name, basedir, opts=None):
+		# Check filter with this option combination was already used:
+		flt = self._filters.get(fltName)
+		if flt:
+			return flt
+		# First time:
+		flt = Filter(None)
+		flt.returnRawHost = True
+		flt.checkAllRegex = True
+		flt.checkFindTime = False
+		flt.active = True
+		# Read filter:
 		if opts is None: opts = dict()
-		# Check filter exists
+		opts = opts.copy()
 		filterConf = FilterReader(name, "jail", opts,
 			basedir=basedir, share_config=unittest.F2B.share_config)
 		self.assertEqual(filterConf.getFile(), name)
@@ -103,25 +110,28 @@ class FilterSamplesRegex(unittest.TestCase):
 				self.fail('Unexpected config-token %r in stream' % (opt,))
 			for optval in optval:
 				if opt[2] == "prefregex":
-					self.filter.prefRegex = optval
+					flt.prefRegex = optval
 				elif opt[2] == "addfailregex":
-					self.filter.addFailRegex(optval)
+					flt.addFailRegex(optval)
 				elif opt[2] == "addignoreregex":
-					self.filter.addIgnoreRegex(optval)
+					flt.addIgnoreRegex(optval)
 				elif opt[2] == "maxlines":
-					self.filter.setMaxLines(optval)
+					flt.setMaxLines(optval)
 				elif opt[2] == "datepattern":
-					self.filter.setDatePattern(optval)
+					flt.setDatePattern(optval)
 
 		# test regexp contains greedy catch-all before <HOST>, that is
 		# not hard-anchored at end or has not precise sub expression after <HOST>:
-		regexList = self.filter.getFailRegex()
+		regexList = flt.getFailRegex()
 		for fr in regexList:
 			if RE_WRONG_GREED.search(fr): # pragma: no cover
 				raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before <HOST>, "
 					"that is not hard-anchored at end or has not precise sub expression after <HOST>:\n%s" %
-					(name, str(fr).replace(RE_HOST, '<HOST>')))
-		return regexList
+					(fltName, str(fr).replace(RE_HOST, '<HOST>')))
+		# Cache within used filter combinations and return:
+		flt = [flt, set()]
+		self._filters[fltName] = flt
+		return flt
 
 def testSampleRegexsFactory(name, basedir):
 	def testFilter(self):
@@ -129,18 +139,11 @@ def testSampleRegexsFactory(name, basedir):
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
 			"No sample log file available for '%s' filter" % name)
-
-		regexList = None
-		regexsUsedIdx = set()
-		regexsUsedRe = set()
+		
 		filenames = [name]
+		regexsUsedRe = set()
 
-		def _testMissingSamples():
-			for failRegexIndex, failRegex in enumerate(regexList):
-				self.assertTrue(
-					failRegexIndex in regexsUsedIdx or failRegex in regexsUsedRe,
-					"Regex for filter '%s' has no samples: %i: %r" %
-						(name, failRegexIndex, failRegex))
+		# process each test-file (note: array filenames can grow during processing):
 		i = 0
 		while i < len(filenames):
 			filename = filenames[i]; i += 1;
@@ -154,13 +157,17 @@ def testSampleRegexsFactory(name, basedir):
 						faildata = json.loads(jsonREMatch.group(2))
 						# filterOptions - dict in JSON to control filter options (e. g. mode, etc.):
 						if jsonREMatch.group(1) == 'filterOptions':
-							# another filter mode - we should check previous also:
-							if self.filter is not None:
-								_testMissingSamples()
-								regexsUsedIdx = set() # clear used indices (possible overlapping by mode change)
-							# read filter with another setting:
-							self.filter = None
-							regexList = self._readFilter(name, basedir, opts=faildata)
+							# following lines with another filter options:
+							self._filterTests = []
+							for opts in (faildata if isinstance(faildata, list) else [faildata]):
+								# unique filter name (using options combination):
+								self.assertTrue(isinstance(opts, dict))
+								fltName = opts.get('filterName')
+								if not fltName: fltName = str(opts) if opts else ''
+								fltName = name + fltName
+								# read it:
+								flt = self._readFilter(fltName, name, basedir, opts=opts)
+								self._filterTests.append((fltName, flt))
 							continue
 						# addFILE - filename to "include" test-files should be additionally parsed:
 						if jsonREMatch.group(1) == 'addFILE':
@@ -176,65 +183,81 @@ def testSampleRegexsFactory(name, basedir):
 				else: # pragma: no cover - normally unreachable
 					faildata = {}
 
-				if self.filter is None:
-					regexList = self._readFilter(name, basedir, opts=None)
+				# if filter options was not yet specified:
+				if not self._filterTests:
+					fltName = name
+					flt = self._readFilter(fltName, name, basedir, opts=None)
+					self._filterTests = [(fltName, flt)]
 
-				try:
-					ret = self.filter.processLine(line)
-					if not ret:
-						# Bypass if filter constraint specified:
-						if faildata.get('filter') and name != faildata.get('filter'):
+				# process line using several filter options (if specified in the test-file):
+				for fltName, flt in self._filterTests:
+					flt, regexsUsedIdx = flt
+					regexList = flt.getFailRegex()
+
+					try:
+						ret = flt.processLine(line)
+						if not ret:
+							# Bypass if filter constraint specified:
+							if faildata.get('filter') and name != faildata.get('filter'):
+								continue
+							# Check line is flagged as none match
+							self.assertFalse(faildata.get('match', True),
+								"Line not matched when should have")
 							continue
-						# Check line is flagged as none match
-						self.assertFalse(faildata.get('match', True),
-							"Line not matched when should have")
-						continue
 
-					failregex, fid, fail2banTime, fail = ret[0]
-					# Bypass no failure helpers-regexp:
-					if not faildata.get('match', False) and (fid is None or fail.get('nofail')):
+						failregex, fid, fail2banTime, fail = ret[0]
+						# Bypass no failure helpers-regexp:
+						if not faildata.get('match', False) and (fid is None or fail.get('nofail')):
+							regexsUsedIdx.add(failregex)
+							regexsUsedRe.add(regexList[failregex])
+							continue
+
+						# Check line is flagged to match
+						self.assertTrue(faildata.get('match', False), 
+							"Line matched when shouldn't have")
+						self.assertEqual(len(ret), 1,
+							"Multiple regexs matched %r" % (map(lambda x: x[0], ret)))
+
+						# Verify match captures (at least fid/host) and timestamp as expected
+						for k, v in faildata.iteritems():
+							if k not in ("time", "match", "desc", "filter"):
+								fv = fail.get(k, None)
+								# Fallback for backwards compatibility (previously no fid, was host only):
+								if k == "host" and fv is None:
+									fv = fid
+								self.assertEqual(fv, v)
+
+						t = faildata.get("time", None)
+						try:
+							jsonTimeLocal =	datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
+						except ValueError:
+							jsonTimeLocal =	datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f")
+
+						jsonTime = time.mktime(jsonTimeLocal.timetuple())
+						
+						jsonTime += jsonTimeLocal.microsecond / 1000000
+
+						self.assertEqual(fail2banTime, jsonTime,
+							"UTC Time  mismatch %s (%s) != %s (%s)  (diff %.3f seconds)" % 
+							(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
+							jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
+							fail2banTime - jsonTime) )
+
 						regexsUsedIdx.add(failregex)
 						regexsUsedRe.add(regexList[failregex])
-						continue
+					except AssertionError as e: # pragma: no cover
+						raise AssertionError("%s: %s on: %s:%i, line:\n%s" % (
+									fltName, e, logFile.filename(), logFile.filelineno(), line))
 
-					# Check line is flagged to match
-					self.assertTrue(faildata.get('match', False), 
-						"Line matched when shouldn't have")
-					self.assertEqual(len(ret), 1,
-						"Multiple regexs matched %r" % (map(lambda x: x[0], ret)))
-
-					# Verify match captures (at least fid/host) and timestamp as expected
-					for k, v in faildata.iteritems():
-						if k not in ("time", "match", "desc", "filter"):
-							fv = fail.get(k, None)
-							# Fallback for backwards compatibility (previously no fid, was host only):
-							if k == "host" and fv is None:
-								fv = fid
-							self.assertEqual(fv, v)
-
-					t = faildata.get("time", None)
-					try:
-						jsonTimeLocal =	datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
-					except ValueError:
-						jsonTimeLocal =	datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f")
-
-					jsonTime = time.mktime(jsonTimeLocal.timetuple())
-					
-					jsonTime += jsonTimeLocal.microsecond / 1000000
-
-					self.assertEqual(fail2banTime, jsonTime,
-						"UTC Time  mismatch %s (%s) != %s (%s)  (diff %.3f seconds)" % 
-						(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
-						jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
-						fail2banTime - jsonTime) )
-
-					regexsUsedIdx.add(failregex)
-					regexsUsedRe.add(regexList[failregex])
-				except AssertionError as e: # pragma: no cover
-					raise AssertionError("%s on: %s:%i, line:\n%s" % (
-								e, logFile.filename(), logFile.filelineno(), line))
-
-		_testMissingSamples()
+		# check missing samples for regex using each filter-options combination:
+		for fltName, flt in self._filters.iteritems():
+			flt, regexsUsedIdx = flt
+			regexList = flt.getFailRegex()
+			for failRegexIndex, failRegex in enumerate(regexList):
+				self.assertTrue(
+					failRegexIndex in regexsUsedIdx or failRegex in regexsUsedRe,
+					"%s: Regex has no samples: %i: %r" %
+						(fltName, failRegexIndex, failRegex))
 
 	return testFilter
 

From aa92b68d4a66cdd3af94f5ec9f6dcc0618d3b043 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 10 Jul 2017 20:49:28 +0200
Subject: [PATCH 435/623] filter.d/postfix.conf: normalized several
 postfix-filters using parameter `mode` (as discussed in gh-1813); introduced
 parameter `mode`: more (default, combines normal and rbl), auth, normal, rbl,
 ddos, extra or aggressive (combines all) replacement for gh-1239, gh-1697,
 gh-1764; closes gh-1245, gh-1297.

---
 config/filter.d/postfix-rbl.conf       | 19 ------
 config/filter.d/postfix-sasl.conf      | 21 ------
 config/filter.d/postfix.conf           | 47 ++++++++++++--
 config/jail.conf                       | 14 ++--
 fail2ban/server/filter.py              | 13 ++--
 fail2ban/tests/files/logs/postfix      | 89 ++++++++++++++++++++++++++
 fail2ban/tests/files/logs/postfix-rbl  |  8 ---
 fail2ban/tests/files/logs/postfix-sasl | 32 ---------
 8 files changed, 150 insertions(+), 93 deletions(-)
 delete mode 100644 config/filter.d/postfix-rbl.conf
 delete mode 100644 config/filter.d/postfix-sasl.conf
 delete mode 100644 fail2ban/tests/files/logs/postfix-rbl
 delete mode 100644 fail2ban/tests/files/logs/postfix-sasl

diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf
deleted file mode 100644
index 0a9078f0..00000000
--- a/config/filter.d/postfix-rbl.conf
+++ /dev/null
@@ -1,19 +0,0 @@
-# Fail2Ban filter for Postfix's RBL based Blocked hosts
-#
-#
-
-[INCLUDES]
-
-# Read common prefixes. If any customizations available -- read them from
-# common.local
-before = common.conf
-
-[Definition]
-
-_daemon = postfix(-\w+)?/smtpd
-
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
-
-ignoreregex =
-
-# Author: Lee Clemens
diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf
deleted file mode 100644
index 1a24ca94..00000000
--- a/config/filter.d/postfix-sasl.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Fail2Ban filter for postfix authentication failures
-#
-
-[INCLUDES]
-
-before = common.conf
-
-[Definition]
-
-_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
-
-failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
-
-ignoreregex = authentication failed: Connection lost to authentication server$
-
-[Init]
-
-journalmatch = _SYSTEMD_UNIT=postfix.service
-
-
-# Author: Yaroslav Halchenko
diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index e6a3e5c2..b86b3d4d 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -13,15 +13,54 @@ before = common.conf
 _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])?
 _port = (?::\d+)?
 
-prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^RCPT from [^[]*\[<HOST>\]<_port>: 55[04] 5\.7\.1\s
-            ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname\b
+mdpr-normal = (?:NOQUEUE: reject:|improper command pipelining after \S+)
+mdre-normal=^RCPT from [^[]*\[<HOST>\]<_port>: 55[04] 5\.7\.1\s
+            ^RCPT from [^[]*\[<HOST>\]<_port>: 45[04] 4\.7\.1 (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b)
             ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b
             ^EHLO from [^[]*\[<HOST>\]<_port>: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b
             ^VRFY from [^[]*\[<HOST>\]<_port>: 550 5\.1\.1\s
             ^RCPT from [^[]*\[<HOST>\]<_port>: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b
-            ^after \S+ from [^[]*\[<HOST>\]:?
+            ^from [^[]*\[<HOST>\]:?
+
+mdpr-auth = warning:
+mdre-auth = ^[^[]*\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed:(?! Connection lost to authentication server| Invalid authentication mechanism)
+mdre-auth2= ^[^[]*\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed:(?! Connection lost to authentication server)
+# todo: check/remove "Invalid authentication mechanism" from ignore list, if gh-1243 will get finished (see gh-1297).
+
+# Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only:
+mdpr-rbl = %(mdpr-normal)s
+mdre-rbl  = ^RCPT from [^[]*\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+
+# Mode "rbl" currently included in mode "normal" (within 1st rule)
+mdpr-more = %(mdpr-normal)s
+mdre-more = %(mdre-normal)s
+
+mdpr-ddos = lost connection after(?! DATA) [A-Z]+
+mdre-ddos = ^from [^[]*\[<HOST>\]:?
+
+mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s)
+mdre-extra = %(mdre-auth)s
+            %(mdre-normal)s
+
+mdpr-aggressive = (?:%(mdpr-auth)s|%(mdpr-normal)s|%(mdpr-ddos)s)
+mdre-aggressive = %(mdre-auth2)s
+                  %(mdre-normal)s
+
+
+
+failregex = <mdre-<mode>>
+
+# Parameter "mode": more (default combines normal and rbl), auth, normal, rbl, ddos, extra or aggressive (combines all)
+# Usage example (for jail.local):
+#   [postfix]
+#   mode = aggressive
+#   # or another jail (rewrite filter parameters of jail):
+#   [postfix-rbl]
+#   filter = postfix[mode=rbl]
+#
+mode = more
 
 ignoreregex = 
 
diff --git a/config/jail.conf b/config/jail.conf
index 21bc898e..71a8332f 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -533,14 +533,17 @@ backend  = %(syslog_backend)s
 
 
 [postfix]
-
-port     = smtp,465,submission
-logpath  = %(postfix_log)s
-backend  = %(postfix_backend)s
+# To use another modes set filter parameter "mode" in jail.local:
+mode    = more
+filter  = postfix[mode=%(mode)s]
+port    = smtp,465,submission
+logpath = %(postfix_log)s
+backend = %(postfix_backend)s
 
 
 [postfix-rbl]
 
+filter   = postfix[mode=rbl]
 port     = smtp,465,submission
 logpath  = %(postfix_log)s
 backend  = %(postfix_backend)s
@@ -624,8 +627,9 @@ logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
 
-[postfix-sasl]
+[postfix-auth]
 
+filter   = postfix[mode=auth]
 port     = smtp,465,submission,imap3,imaps,pop3,pop3s
 # You might consider monitoring /var/log/mail.warn instead if you are
 # running postfix since it would provide the same log lines at the
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index a8f99998..e0a9340e 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -668,16 +668,19 @@ class Filter(JailThread):
 				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
 		else:
 			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
-		logSys.log(5, "Looking for failregex match of %r", self.__lineBuffer)
+		logSys.log(5, "Looking for match of %r", self.__lineBuffer)
 
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
+			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG:
+				logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
 			self.__prefRegex.search(self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
+				logSys.log(5, "  Prefregex not matched")
 				return failList
 			preGroups = self.__prefRegex.getGroups()
-			logSys.log(7, "Pre-filter matched %s", preGroups)
+			logSys.log(7, "  Pre-filter matched %s", preGroups)
 			repl = preGroups.get('content')
 			# Content replacement:
 			if repl:
@@ -686,17 +689,19 @@ class Filter(JailThread):
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
+			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG:
+				logSys.log(5, "  Looking for failregex %r", failRegex.getRegex())
 			failRegex.search(self.__lineBuffer, orgBuffer)
 			if not failRegex.hasMatched():
 				continue
 			# The failregex matched.
-			logSys.log(7, "Matched %s", failRegex)
+			logSys.log(7, "  Matched %s", failRegex)
 			# Checks if we must ignore this match.
 			if self.ignoreLine(failRegex.getMatchedTupleLines()) \
 					is not None:
 				# The ignoreregex matched. Remove ignored match.
 				self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-				logSys.log(7, "Matched ignoreregex and was ignored")
+				logSys.log(7, "  Matched ignoreregex and was ignored")
 				if not self.checkAllRegex:
 					break
 				else:
diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix
index 7f24cdd0..54b8be99 100644
--- a/fail2ban/tests/files/logs/postfix
+++ b/fail2ban/tests/files/logs/postfix
@@ -1,3 +1,5 @@
+# filterOptions: [{}, {"mode": "normal"}, {"mode": "aggressive"}]
+
 # per https://github.com/fail2ban/fail2ban/issues/125
 # and https://github.com/fail2ban/fail2ban/issues/126
 # failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" }
@@ -45,5 +47,92 @@ Jun 12 08:58:35 xxx postfix/smtpd[2931]: NOQUEUE: reject: RCPT from unknown[1.2.
 # failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" }
 Jun 12 08:58:35 xxx postfix/smtpd[13533]: improper command pipelining after AUTH from unknown[1.2.3.4]: QUIT
 
+# ---------------------------------------
+# Test-cases of postfix-postscreen:
+# ---------------------------------------
+
 # failJSON: { "time": "2005-05-05T15:51:11", "match": true , "host": "216.245.194.173", "desc": "postfix postscreen / gh-1764" }
 May  5 15:51:11 xxx postfix/postscreen[1148]: NOQUEUE: reject: RCPT from [216.245.194.173]:60591: 550 5.7.1 Service unavailable; client [216.245.194.173] blocked using rbl.example.com; from=<spammer@example.com>, to=<goodguy@example.com>, proto=ESMTP, helo=<badguy.example.com>
+
+# ---------------------------------------
+# Test-cases of postfix-rbl:
+# ---------------------------------------
+# filterOptions: [{}, {"mode": "rbl"}, {"mode": "aggressive"}]
+
+# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
+Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
+
+# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
+Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
+
+# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" }
+Feb  7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from=<spammer@example.com> to=<goodguy@example.com> proto=SMTP helo=<WIN-5N8GBBS0R5I>
+
+# ---------------------------------------
+# Test-cases of postfix-sasl:
+# ---------------------------------------
+# filterOptions: [{"mode": "auth"}, {"mode": "aggressive"}]
+
+#1 Example from postfix from dbts #507990
+# failJSON: { "time": "2004-12-02T22:24:22", "match": true , "host": "114.44.142.233" }
+Dec  2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.net[114.44.142.233]: SASL CRAM-MD5 authentication failed: PDc3OTEwNTkyNTEyMzA2NDIuMTIyODI1MzA2MUBoZWw+
+#2 Example from postfix from dbts #573314
+# failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" }
+Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure
+
+#3 Example from postfix post-debian changes to rename to add "submission" to syslog name
+# failJSON: { "time": "2004-09-06T00:44:56", "match": true , "host": "82.221.106.233" }
+Sep  6 00:44:56 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL LOGIN authentication failed: UGFzc3dvcmQ6
+
+#4 Example from postfix post-debian changes to rename to add "submission" to syslog name + downcase
+# failJSON: { "time": "2004-09-06T00:44:57", "match": true , "host": "82.221.106.233" }
+Sep  6 00:44:57 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL login authentication failed: UGFzc3dvcmQ6
+
+#5 Example to add :
+# failJSON: { "time": "2005-01-29T08:11:45", "match": true , "host": "1.1.1.1" }
+Jan 29 08:11:45 mail postfix/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Password:
+
+# failJSON: { "time": "2005-01-29T08:11:45", "match": true , "host": "1.1.1.1" }
+Jan 29 08:11:45 mail postfix-incoming/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Password:
+
+# failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" }
+Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: warning: astra4139.startdedicated.de[62.138.2.143]: SASL LOGIN authentication failed: UGFzc3dvcmQ6
+
+# failJSON: { "time": "2005-08-03T15:30:49", "match": true , "host": "98.191.84.74" }
+Aug 3 15:30:49 ksusha postfix/smtpd[17041]: warning: mail.foldsandwalker.com[98.191.84.74]: SASL Plain authentication failed:
+
+# failJSON: { "time": "2004-11-04T09:11:01", "match": true , "host": "192.0.2.150", "desc": "without reason for fail, see gh-1245" }
+Nov  4 09:11:01 mail postfix/submission/smtpd[27133]: warning: unknown[192.0.2.150]: SASL PLAIN authentication failed:
+
+#6 Example to ignore because due to a failed attempt to connect to authentication service - no malicious activities whatsoever
+# failJSON: { "match": false }
+Feb  3 08:29:28 mail postfix/smtpd[21022]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Connection lost to authentication server
+
+# filterOptions: [{"mode": "auth"}]
+
+# failJSON: { "match": false, "desc": "not aggressive" }
+Jan 14 16:18:16 xxx postfix/smtpd[14933]: warning: host[192.0.2.5]: SASL CRAM-MD5 authentication failed: Invalid authentication mechanism
+
+# filterOptions: [{"mode": "aggressive"}]
+
+# failJSON: { "time": "2005-01-14T16:18:16", "match": true , "host": "192.0.2.5", "desc": "aggressive only" }
+Jan 14 16:18:16 xxx postfix/smtpd[14933]: warning: host[192.0.2.5]: SASL CRAM-MD5 authentication failed: Invalid authentication mechanism
+
+# ---------------------------------------
+# Test-cases of postfix DDOS mode:
+# ---------------------------------------
+
+# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}]
+
+# failJSON: { "time": "2005-02-18T09:45:10", "match": true , "host": "192.0.2.10" }
+Feb 18 09:45:10 xxx postfix/smtpd[42]: lost connection after CONNECT from spammer.example.com[192.0.2.10]
+# failJSON: { "time": "2005-02-18T09:45:12", "match": true , "host": "192.0.2.42" }
+Feb 18 09:45:12 xxx postfix/smtpd[42]: lost connection after STARTTLS from spammer.example.com[192.0.2.42]
+# failJSON: { "time": "2005-02-18T09:48:04", "match": true , "host": "192.0.2.23" }
+Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[192.0.2.23]
+# failJSON: { "time": "2005-02-18T09:48:04", "match": true , "host": "192.0.2.23" }
+Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[192.0.2.23]
+
+# filterOptions: [{}, {"mode": "ddos"}, {"mode": "aggressive"}]
+# failJSON: { "match": false, "desc": "don't affect lawful data (sporadical connection aborts within DATA-phase, see gh-1813 for discussion)" }
+Feb 18 09:50:05 xxx postfix/smtpd[42]: lost connection after DATA from good-host.example.com[192.0.2.10]
diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl
deleted file mode 100644
index 6aeac03b..00000000
--- a/fail2ban/tests/files/logs/postfix-rbl
+++ /dev/null
@@ -1,8 +0,0 @@
-# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
-Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
-
-# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
-Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
-
-# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" }
-Feb  7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from=<spammer@example.com> to=<goodguy@example.com> proto=SMTP helo=<WIN-5N8GBBS0R5I>
diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl
deleted file mode 100644
index cdcb5121..00000000
--- a/fail2ban/tests/files/logs/postfix-sasl
+++ /dev/null
@@ -1,32 +0,0 @@
-#1 Example from postfix from dbts #507990
-# failJSON: { "time": "2004-12-02T22:24:22", "match": true , "host": "114.44.142.233" }
-Dec  2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.net[114.44.142.233]: SASL CRAM-MD5 authentication failed: PDc3OTEwNTkyNTEyMzA2NDIuMTIyODI1MzA2MUBoZWw+
-#2 Example from postfix from dbts #573314
-# failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" }
-Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure
-
-#3 Example from postfix post-debian changes to rename to add "submission" to syslog name
-# failJSON: { "time": "2004-09-06T00:44:56", "match": true , "host": "82.221.106.233" }
-Sep  6 00:44:56 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL LOGIN authentication failed: UGFzc3dvcmQ6
-
-#4 Example from postfix post-debian changes to rename to add "submission" to syslog name + downcase
-# failJSON: { "time": "2004-09-06T00:44:57", "match": true , "host": "82.221.106.233" }
-Sep  6 00:44:57 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL login authentication failed: UGFzc3dvcmQ6
-
-#5 Example to add :
-# failJSON: { "time": "2005-01-29T08:11:45", "match": true , "host": "1.1.1.1" }
-Jan 29 08:11:45 mail postfix/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Password:
-
-#6 Example to ignore because due to a failed attempt to connect to authentication service - no malicious activities whatsoever
-# failJSON: { "time": "2005-02-03T08:29:28", "match": false , "host": "1.1.1.1" }
-Feb  3 08:29:28 mail postfix/smtpd[21022]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Connection lost to authentication server
-
-# failJSON: { "time": "2005-01-29T08:11:45", "match": true , "host": "1.1.1.1" }
-Jan 29 08:11:45 mail postfix-incoming/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Password:
-
-# failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" }
-Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: warning: astra4139.startdedicated.de[62.138.2.143]: SASL LOGIN authentication failed: UGFzc3dvcmQ6
-
-# failJSON: { "time": "2005-08-03T15:30:49", "match": true , "host": "98.191.84.74" }
-Aug 3 15:30:49 ksusha postfix/smtpd[17041]: warning: mail.foldsandwalker.com[98.191.84.74]: SASL Plain authentication failed:
-

From e26cc5de45ec256d519b8b0c3bde533027303b79 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 11:57:48 +0200
Subject: [PATCH 436/623] restore backwards compatibility (jail postfix-sasl);
 changelog update

---
 ChangeLog        | 8 +++++++-
 config/jail.conf | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 774bfcf1..50d75f29 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -26,8 +26,14 @@ TODO: implementing of options resp. other tasks from PR #1346
   - rewritten using `prefregex` and used MLFID-related multi-line parsing;
   - optional parameter `mode` introduced: normal (default), extra or aggressive
 * `filter.d/haproxy-http-auth`: do not mistake client port for part of an IPv6 address (gh-1745)
-* `filter.d/postfix-sasl.conf`
+* `filter.d/postfix.conf`:
     - updated to latest postfix formats
+    - joined several postfix filter together (normalized and optimized version, gh-1825)
+    - introduced new parameter `mode` (see gh-1825): more (default, combines normal and rbl), auth, normal,
+      rbl, ddos, extra or aggressive (combines all)
+    - postfix postscreen (resp. other RBL's compatibility fix, gh-1764, gh-1825)
+* `filter.d/postfix-rbl.conf`: removed (replaced with `postfix[mode=rbl]`)
+* `filter.d/postfix-sasl.conf`: removed (replaced with `postfix[mode=auth]`)
 * `filter.d/roundcube-auth.conf`:
     - fixed regex when logging authentication errors to journal instead to a local file (gh-1159);
     - additionally fixed more complex injections on username (e. g. using dot after fake host).
diff --git a/config/jail.conf b/config/jail.conf
index 71a8332f..e3e89ff0 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -627,7 +627,7 @@ logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
 
-[postfix-auth]
+[postfix-sasl]
 
 filter   = postfix[mode=auth]
 port     = smtp,465,submission,imap3,imaps,pop3,pop3s

From cf3b8f63f6b1564fef26f9e98801feeb29445a4d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 12:16:12 +0200
Subject: [PATCH 437/623] coverage fix

---
 fail2ban/server/filter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e0a9340e..c4f29878 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -673,7 +673,7 @@ class Filter(JailThread):
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG:
+			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
 				logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
 			self.__prefRegex.search(self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
@@ -689,7 +689,7 @@ class Filter(JailThread):
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG:
+			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
 				logSys.log(5, "  Looking for failregex %r", failRegex.getRegex())
 			failRegex.search(self.__lineBuffer, orgBuffer)
 			if not failRegex.hasMatched():

From 89f2dbb97bca0b7c46324e17d218d266faa3c18d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 14:01:42 +0200
Subject: [PATCH 438/623] small bug fix (missing `-` by option `--timeout`,
 wrong module reference)

---
 fail2ban/client/fail2bancmdline.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index a9387bc6..3a3cedcd 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -155,8 +155,8 @@ class Fail2banCmdLine():
 				self._conf["background"] = False
 			elif o == "--async":
 				self._conf["async"] = True
-			elif o == "-timeout":
-				from ..mytime import MyTime
+			elif o == "--timeout":
+				from ..server.mytime import MyTime
 				self._conf["timeout"] = MyTime.str2seconds(opt[1])
 			elif o in ["-h", "--help"]:
 				self.dispUsage()

From 00456b8270aed2a0c1c2433f19c676ef52fe938d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 14:00:15 +0200
Subject: [PATCH 439/623] review: documentation, small enhancement of
 `fail2ban-client` to test time abbreviation format:

fail2ban-client --str2sec 1d12h30m
---
 fail2ban/client/fail2bancmdline.py       |  7 ++++-
 fail2ban/tests/fail2banclienttestcase.py |  3 ++
 man/jail.conf.5                          | 36 ++++++++++++++++++++++--
 3 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 3a3cedcd..f213d037 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -111,6 +111,7 @@ class Fail2banCmdLine():
 		output("    -f                      start server in foreground")
 		output("    --async                 start server in async mode (for internal usage only, don't read configuration)")
 		output("    --timeout               timeout to wait for the server (for internal usage only, don't read configuration)")
+		output("    --str2sec <STRING>      convert time abbreviation format to seconds")
 		output("    -h, --help              display this help message")
 		output("    -V, --version           print the version")
 
@@ -158,6 +159,10 @@ class Fail2banCmdLine():
 			elif o == "--timeout":
 				from ..server.mytime import MyTime
 				self._conf["timeout"] = MyTime.str2seconds(opt[1])
+			elif o == "--str2sec":
+				from ..server.mytime import MyTime
+				output(MyTime.str2seconds(opt[1]))
+				return True
 			elif o in ["-h", "--help"]:
 				self.dispUsage()
 				return True
@@ -179,7 +184,7 @@ class Fail2banCmdLine():
 			# Reads the command line options.
 			try:
 				cmdOpts = 'hc:s:p:xfbdtviqV'
-				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'timeout=', 'help', 'version']
+				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'timeout=', 'str2sec=', 'help', 'version']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 3e046042..caacf63c 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -416,6 +416,9 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.pruneLog()
 		self.execSuccess((), "-vq", "-V")
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
+		self.pruneLog()
+		self.execSuccess((), "--str2sec", "1d12h30m")
+		self.assertLogged("131400")
 
 	@with_tmpdir
 	def testClientDump(self, tmp):
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index a43346e1..a8afe53c 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -231,10 +231,10 @@ IP will not be banned if command returns successfully (exit code 0).
 Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution. Currently only <ip> is supported however more will be added later.
 .TP
 .B bantime
-effective ban duration (in seconds).
+effective ban duration (in seconds or time abbreviation format).
 .TP
 .B findtime
-time interval (in seconds) before the current time where failures will count towards a ban.
+time interval (in seconds or time abbreviation format) before the current time where failures will count towards a ban.
 .TP
 .B maxretry
 number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP.
@@ -278,6 +278,38 @@ action = smtp.py[dest=chris@example.com, actname=smtp-chris]
          smtp.py[dest=sally@example.com, actname=smtp-sally]
 .fi
 
+.SH "TIME ABBREVIATION FORMAT"
+The time entries in fail2ban configuration (like \fBfindtime\fR or \fBbantime\fR) can be provided as integer in seconds or as string using special abbreviation format (e. g. \fB600\fR is the same as \fB10m\fR).
+
+.TP
+.B Abbreviation tokens:
+
+.RS
+.nf
+years?, yea?, yy?
+months?, mon?
+weeks?, wee?, ww?
+days?, da, dd?
+hours?, hou?, hh?
+minutes?, min?, mm?
+seconds?, sec?, ss?
+
+The question mark (?) means the optional character, so \fBday\fR as well as \fBdays\fR can be used.
+.fi
+.RE
+
+You can combine multiple tokens in format (separated with space resp. without separator), e. g.: \fB1y 6mo\fR or \fB1d12h30m\fR.
+.br
+Note that tokens \fBm\fR as well as \fBmm\fR means minutes, for month use abbreviation \fBmo\fR or \fBmon\fR.
+
+The time format can be tested using \fBfail2ban-client\fR:
+
+.RS
+.nf
+fail2ban-client --str2sec 1d12h
+.fi
+.RE
+
 .SH "ACTION CONFIGURATION FILES (\fIaction.d/*.conf\fB)"
 Action files specify which commands are executed to ban and unban an IP address.
 

From dae4988aea85e3d63516778c5f93ca38cae16cef Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 14:59:24 +0200
Subject: [PATCH 440/623] filter.d/roundcube-auth.conf: fixes failregex not
 working with `X-Real-IP` or/and `X-Forwarded-For` (gh-1303)

---
 config/filter.d/roundcube-auth.conf      | 2 +-
 fail2ban/tests/files/logs/roundcube-auth | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf
index 1d0007da..9912ff47 100644
--- a/config/filter.d/roundcube-auth.conf
+++ b/config/filter.d/roundcube-auth.conf
@@ -15,7 +15,7 @@ before = common.conf
 
 prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+>)? IMAP Error)?: <F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
+failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(?:(?:\([^\)]*\))?\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
             ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
 
 ignoreregex = 
diff --git a/fail2ban/tests/files/logs/roundcube-auth b/fail2ban/tests/files/logs/roundcube-auth
index 8c491b38..f3f762d2 100644
--- a/fail2ban/tests/files/logs/roundcube-auth
+++ b/fail2ban/tests/files/logs/roundcube-auth
@@ -49,5 +49,8 @@ Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 12
 # failJSON: { "time": "2015-05-10T19:02:52", "match": true , "host": "1.2.3.4" }
 [10-May-2015 13:02:52 -0400]: <4z506z6r> Failed login for admin@example.com from 1.2.3.4 in session 4z506z6rvddstv6k7jz08hxo27 (error: 0)
 
+# failJSON: { "time": "2016-01-17T10:09:06", "match": true , "host": "192.0.2.2", "desc": "X-Real-IP or/and X-Forwarded-For in-between, gh-1303"}
+[17-Jan-2016 11:09:06 +0200]: <j2pncet8> IMAP Error: Login failed for user@example.com from 192.0.2.2(X-Real-IP: 192.0.2.3,X-Forwarded-For: 192.0.2.4). AUTHENTICATE PLAIN: Authentication failed. in /usr/share/roundcubemail/program/lib/Roundcube/rcube_imap.php on line 197 (POST /?_task=login?_task=login&_action=login)
+
 # failJSON: { "time": "2005-05-19T06:07:48", "match": true , "host": "192.0.2.1", "desc": "Roundcube logged to journald instead to a local file."}
 May 19 06:07:48 server roundcube[21296]: <crk9n97i> IMAP Error: Login failed for test from 192.0.2.1. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login)

From ea3a6aa9713a42acd0716b136fde4287a8d5fcbe Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 15:02:59 +0200
Subject: [PATCH 441/623] ChangeLog updated

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 50d75f29..671602be 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,6 +35,7 @@ TODO: implementing of options resp. other tasks from PR #1346
 * `filter.d/postfix-rbl.conf`: removed (replaced with `postfix[mode=rbl]`)
 * `filter.d/postfix-sasl.conf`: removed (replaced with `postfix[mode=auth]`)
 * `filter.d/roundcube-auth.conf`:
+    - fixed regex when `X-Real-IP` or/and `X-Forwarded-For` are present after host (gh-1303);
     - fixed regex when logging authentication errors to journal instead to a local file (gh-1159);
     - additionally fixed more complex injections on username (e. g. using dot after fake host).
 * `action.d/complain.conf`

From 7217ef5c9e33541baf3fe61421ab9ce2077252dd Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 15:25:51 +0200
Subject: [PATCH 442/623] filter.d/ejabberd-auth.conf: fixed ejabberd filter -
 accept new log-format with `wait_for_sasl_response` instead of
 `wait_for_feature_request` + optional part "IP " (gh-993)

---
 config/filter.d/ejabberd-auth.conf      | 4 ++--
 fail2ban/tests/files/logs/ejabberd-auth | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf
index 56517489..de27c1cd 100644
--- a/config/filter.d/ejabberd-auth.conf
+++ b/config/filter.d/ejabberd-auth.conf
@@ -16,8 +16,8 @@
 #          searched for other failures. This tag can be used multiple times.
 # Values:  TEXT
 #
-failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\) Failed authentication for .+ from IP <HOST> \({{(?:\d+,){3}\d+},\d+}\)$
-            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:wait_for_feature_request:\d+ \([^\)]+\) Failed authentication for \S+ from IP <HOST>$
+failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\) Failed authentication for \S+ from (?:IP )?<HOST>(?: \({{(?:\d+,){3}\d+},\d+}\))?$
+            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:\w+:\d+ \([^\)]+\) Failed authentication for \S+ from (?:IP )?<HOST>$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
diff --git a/fail2ban/tests/files/logs/ejabberd-auth b/fail2ban/tests/files/logs/ejabberd-auth
index e8e89fb8..cc04883f 100644
--- a/fail2ban/tests/files/logs/ejabberd-auth
+++ b/fail2ban/tests/files/logs/ejabberd-auth
@@ -9,3 +9,8 @@ I(<0.370.0>:ejabberd_listener:281) : (#Port<0.6910>) Accepted connection {{192,0
 I(<0.1440.0>:ejabberd_c2s:813) : ({socket_state,tls,{tlssock,#Port<0.6910>,#Port<0.6912>},<0.1439.0>}) Failed authentication for user@example.com from IP 192.0.2.4 ({{192,0,2,4},12716})
 # failJSON: { "time": "2014-01-07T18:09:08", "match": true , "host": "1.2.3.4" }
 2014-01-07 18:09:08.512 [info] <0.22741.1>@ejabberd_c2s:wait_for_feature_request:662 ({socket_state,p1_tls,{tlssock,#Port<0.24718>,#Port<0.24720>},<0.22740.1>}) Failed authentication for test@example.com from IP 1.2.3.4
+
+# new format:
+
+# failJSON: { "time": "2015-03-19T13:57:35", "match": true , "host": "192.0.2.6" }
+2015-03-19 13:57:35.805 [info] <0.585.0>@ejabberd_c2s:wait_for_sasl_response:965 ({socket_state,p1_tls,{tlssock,#Port<0.6434>,#Port<0.6436>},<0.584.0>}) Failed authentication for robin@example.com from 192.0.2.6
\ No newline at end of file

From c9385a2e04f6636256e7d7227ab48f2459f32fc3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 11 Jul 2017 15:28:04 +0200
Subject: [PATCH 443/623] ChangeLog updated

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 671602be..3dd77f01 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -38,6 +38,7 @@ TODO: implementing of options resp. other tasks from PR #1346
     - fixed regex when `X-Real-IP` or/and `X-Forwarded-For` are present after host (gh-1303);
     - fixed regex when logging authentication errors to journal instead to a local file (gh-1159);
     - additionally fixed more complex injections on username (e. g. using dot after fake host).
+* `filter.d/ejabberd-auth.conf`: fixed failregex - accept new log-format (gh-993)
 * `action.d/complain.conf`
   - fixed using new tag `<ip-rev>` (sh/dash compliant now)
 * `action.d/sendmail-geoip-lines.conf`

From b662cf03ac6a2ed86083e180883002922da54c7a Mon Sep 17 00:00:00 2001
From: benrubson <ben.rubson@gmail.com>
Date: Fri, 17 Feb 2017 08:11:33 +0100
Subject: [PATCH 444/623] Apache, detect syslog prefix, simple example

---
 config/filter.d/apache-common.conf | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf
index 3d1f902b..b8c08ab2 100644
--- a/config/filter.d/apache-common.conf
+++ b/config/filter.d/apache-common.conf
@@ -1,6 +1,13 @@
 # Generic configuration items (to be used as interpolations) in other
 # apache filters.
 
+# If you use "ErrorLog syslog" in your Apache configuration,
+# you should then create the following apache-common.local file :
+# [INCLUDES]
+# before = common.conf
+# [DEFAULT]
+# _apache_prefix = %(__prefix_line)s
+
 [INCLUDES]
 
 # Load customizations if any available
@@ -8,7 +15,9 @@ after = apache-common.local
 
 [DEFAULT]
 
-_apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
+_apache_prefix = \[\]\s
+
+_apache_error_client = ^%(_apache_prefix)s\[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
 
 datepattern = {^LN-BEG}
 

From 10cc7e6e599e588cbd0d4538a21dec342b02763e Mon Sep 17 00:00:00 2001
From: benrubson <ben.rubson@gmail.com>
Date: Fri, 17 Feb 2017 08:42:40 +0100
Subject: [PATCH 445/623] Apache, detect syslog prefix, add test

---
 fail2ban/tests/files/logs/apache-auth | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth
index 29de57eb..6e41bab4 100644
--- a/fail2ban/tests/files/logs/apache-auth
+++ b/fail2ban/tests/files/logs/apache-auth
@@ -11,6 +11,10 @@
 # failJSON: { "time": "2013-07-11T01:21:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
 [Thu Jul 11 01:21:44 2013] [error] [client 2606:2800:220:1:248:1893:25c8:1946] user test-ipv6 not found: /
 
+# Must mach syslog prefix
+# failJSON: { "time": "2005-02-16T22:32:48", "match": true , "host": "127.0.0.1" }
+Feb 16 22:32:48 srv httpd[22034]: [error] [client 127.0.0.1] user wrongusername not found: /basic/file
+
 # The failures below use the configuration described in fail2ban/tests/files/config/apache-auth
 #
 

From b561af45ef29ab8f40061c49db9b2de2930b7b22 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 12 Jul 2017 11:25:58 +0200
Subject: [PATCH 446/623] apache-common.conf: introduced parameter `logging`
 for possibility to match lines, if apache logs into syslog/systemd journal;
 added test cases to cover `apache-auth[logging=syslog]`.

---
 config/filter.d/apache-common.conf    | 25 ++++++++++++++++---------
 fail2ban/tests/files/logs/apache-auth | 16 ++++++++++++----
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf
index b8c08ab2..5f3e271d 100644
--- a/config/filter.d/apache-common.conf
+++ b/config/filter.d/apache-common.conf
@@ -1,23 +1,30 @@
 # Generic configuration items (to be used as interpolations) in other
 # apache filters.
 
-# If you use "ErrorLog syslog" in your Apache configuration,
-# you should then create the following apache-common.local file :
-# [INCLUDES]
-# before = common.conf
-# [DEFAULT]
-# _apache_prefix = %(__prefix_line)s
-
 [INCLUDES]
 
+before = common.conf
 # Load customizations if any available
 after = apache-common.local
 
 [DEFAULT]
 
-_apache_prefix = \[\]\s
+# Apache logging mode:
+#   all - universal prefix (logfile, syslog)
+#   logfile - logfile only
+#   syslog - syslog only
+# Use `filter = apache-auth[logging=syslog]` to get more precise regex if apache logs into syslog (ErrorLog syslog).
+# Use `filter = apache-auth[logging=all]` to get universal regex matches both logging variants.
+logging = logfile
 
-_apache_error_client = ^%(_apache_prefix)s\[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
+# Apache logging prefixes (date-pattern prefix, server, process etc.):
+apache-prefix-syslog = %(__prefix_line)s
+apache-prefix-logfile = \[\]\s
+apache-prefix-all = (?:%(apache-prefix-logfile)s|%(apache-prefix-syslog)s)?
+
+apache-prefix = <apache-prefix-<logging>>
+
+_apache_error_client = <apache-prefix>\[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
 
 datepattern = {^LN-BEG}
 
diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth
index 6e41bab4..d430e291 100644
--- a/fail2ban/tests/files/logs/apache-auth
+++ b/fail2ban/tests/files/logs/apache-auth
@@ -11,10 +11,6 @@
 # failJSON: { "time": "2013-07-11T01:21:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
 [Thu Jul 11 01:21:44 2013] [error] [client 2606:2800:220:1:248:1893:25c8:1946] user test-ipv6 not found: /
 
-# Must mach syslog prefix
-# failJSON: { "time": "2005-02-16T22:32:48", "match": true , "host": "127.0.0.1" }
-Feb 16 22:32:48 srv httpd[22034]: [error] [client 127.0.0.1] user wrongusername not found: /basic/file
-
 # The failures below use the configuration described in fail2ban/tests/files/config/apache-auth
 #
 
@@ -128,3 +124,15 @@ Feb 16 22:32:48 srv httpd[22034]: [error] [client 127.0.0.1] user wrongusername
 
 # failJSON: { "time": "2013-11-18T22:39:33", "match": true , "host": "91.49.82.139" }
 [Mon Nov 18 22:39:33 2013] [error] [client 91.49.82.139] user gg not found: /, referer: http://sj.hopto.org/management.html
+
+# filterOptions: {"logging": "syslog"}
+
+# failJSON: { "time": "2005-02-15T16:23:00", "match": true , "host": "192.0.2.1", "desc": "using syslog (ErrorLog syslog)" }
+Feb 15 16:23:00 srv httpd[22034]: [authz_core:error] [pid 22034] [client 192.0.2.1:58585] AH01630: client denied by server configuration: /home/www/
+# failJSON: { "time": "2005-02-15T16:23:40", "match": true , "host": "192.0.2.2", "desc": "using syslog (ErrorLog syslog)" }
+Feb 15 16:23:40 srv httpd/backend1[22034]: [authz_core:error] [pid 22036] [client 192.0.2.2:59392] AH01630: client denied by server configuration: /home/backend1/
+# failJSON: { "time": "2005-02-15T16:54:53", "match": true , "host": "192.0.2.3", "desc": "using syslog (ErrorLog syslog)" }
+Feb 15 16:54:53 tools apache2[18154]: [:error] [pid 18154:tid 140680873617152] [client 192.0.2.3:48154] AH01630: client denied by server configuration: /var/www
+
+# failJSON: { "time": "2005-02-16T22:32:48", "match": true , "host": "127.0.0.1" }
+Feb 16 22:32:48 srv httpd[22034]: [error] [client 127.0.0.1] user wrongusername not found: /basic/file

From 0e3312512907e7d596701f3ca8f28af97bbfefaf Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 12 Jul 2017 11:59:02 +0200
Subject: [PATCH 447/623] be more precise using common `__prefix_line`
 expression (set `_daemon` to recognize apache and httpd only)

---
 config/filter.d/apache-common.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf
index 5f3e271d..3eec83d0 100644
--- a/config/filter.d/apache-common.conf
+++ b/config/filter.d/apache-common.conf
@@ -22,6 +22,9 @@ apache-prefix-syslog = %(__prefix_line)s
 apache-prefix-logfile = \[\]\s
 apache-prefix-all = (?:%(apache-prefix-logfile)s|%(apache-prefix-syslog)s)?
 
+# Setting for __prefix_line (only `logging=syslog`):
+_daemon = (?:apache\d*|httpd(?:/\w+)?)
+
 apache-prefix = <apache-prefix-<logging>>
 
 _apache_error_client = <apache-prefix>\[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]

From a12ac4242b498ad5be7f56da826315a9bd510d79 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 12 Jul 2017 11:59:42 +0200
Subject: [PATCH 448/623] ChangeLog updated

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3dd77f01..d7e77ff3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,12 @@ TODO: implementing of options resp. other tasks from PR #1346
 ### Fixes
 * `filter.d/apache-auth.conf`:
   - better failure recognition using short form of regex (url/referer are foreign inputs, see gh-1645)
+* `filter.d/apache-common.conf` (`filter.d/apache-*.conf`):
+  - support of apache log-format if logging into syslog/systemd (gh-1695), using parameter `logging`,
+    parameter usage for jail:
+      filter = apache-auth[logging=syslog]
+    parameter usage for `apache-common.local`:
+      logging = syslog
 * `filter.d/pam-generic.conf`:
   - [grave] injection on user name to host fixed
 * `filter.d/sshd.conf`:

From 4c0c7b97c06f55c5b35b2c2c6da01874b7191579 Mon Sep 17 00:00:00 2001
From: Kirill <greatorc@yandex.ru>
Date: Mon, 16 May 2016 00:15:45 +0300
Subject: [PATCH 449/623] Update asterisk.conf to new log message

I got an issue like this:
[2016-05-15 22:53:00] SECURITY[26428] res_security_log.c: SecurityEvent="FailedACL",EventTV="2016-05-15T22:53:00.203+0300",Severity="Error",Service="AMI",EventVersion="1",AccountID="admin",SessionID="0x7fb580001518",LocalAddress="IPV4/TCP/0.0.0.0/5038",RemoteAddress="IPV4/TCP/78.129.227.4/62389",SessionTV="1970-01-01T03:00:00.000+0300"

# [sebres] rebased to current master and resolving conflicts.
---
 config/filter.d/asterisk.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 97178002..eba9dd8f 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -25,7 +25,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
             ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
             ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>|[\w]+)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?(,SessionTV="([\d-]+|%(iso8601)s)")?$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
             ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 

From 2ea22b9d303314bd4da977aba3e800f5b3391a88 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 18 Jul 2017 15:46:53 +0200
Subject: [PATCH 450/623] test coverage for gh-1427

---
 fail2ban/tests/files/logs/asterisk | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index f3ece02e..2769cd4c 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -70,6 +70,9 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # failJSON: { "time": "2016-01-28T10:34:33", "match": true , "host": "1.2.3.4" }
 [2016-01-28 10:34:33] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '' rejected because extension not found in context 'my-context'.
 
+# failJSON: { "time": "2016-05-15T22:53:00", "match": true , "host": "192.0.2.4" }
+[2016-05-15 22:53:00] SECURITY[26428] res_security_log.c: SecurityEvent="FailedACL",EventTV="2016-05-15T22:53:00.203+0300",Severity="Error",Service="AMI",EventVersion="1",AccountID="admin",SessionID="0x7fb580001518",LocalAddress="IPV4/TCP/0.0.0.0/5038",RemoteAddress="IPV4/TCP/192.0.2.4/62389",SessionTV="1970-01-01T03:00:00.000+0300"
+
 # Failed authentication with pjsip on Asterisk 13+
 # failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - No matching endpoint found

From 098abae4e63521211388f6d2a2b973ea4e89b6f3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 18 Jul 2017 16:09:53 +0200
Subject: [PATCH 451/623] Remove greedy catch-all before `<HOST>`, make regex
 more universal, fewer prone to errors (should avoid future changes, if some
 optional parameters coming again before/after `RemoteAddress`) + non-captured
 groups now. Test for possible injection (5.6.7.8 in session-id) already
 available, line 59 (thus already covered).

---
 config/filter.d/asterisk.conf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index eba9dd8f..d773a792 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -18,14 +18,14 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 # All Asterisk log messages begin like this:
 log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)?
 
-failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
             ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
             ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed to authenticate
             ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
             ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
+            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (?:user|device) [^@]+@<HOST>\S*$
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>|[\w]+)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?(,SessionTV="([\d-]+|%(iso8601)s)")?$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
             ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 

From a5b62a7f36b41c0bcc94575f7fb6b08310bf7d7a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 18 Jul 2017 16:34:08 +0200
Subject: [PATCH 452/623] failregex extended and simplified (partially ported
 from gh-1409).

---
 config/filter.d/asterisk.conf      |  6 ++++--
 fail2ban/tests/files/logs/asterisk | 21 ++++++++++++++++++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index d773a792..cd3ea150 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -20,15 +20,17 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\
 
 failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
             ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed to authenticate
+            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b)
             ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
             ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (?:user|device) [^@]+@<HOST>\S*$
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
             ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 
+# FreePBX (todo: make optional in v.0.10):
+#            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$
+
 ignoreregex =
 
 
diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 2769cd4c..5e846111 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -89,4 +89,23 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 [2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found
 
 # failJSON: { "time": "2016-06-09T00:01:02", "match": true , "host": "192.0.2.1" }
-[2016-06-09 00:01:02] NOTICE [22382] manager.c: 192.0.2.1 failed to authenticate as 'admin'
\ No newline at end of file
+[2016-06-09 00:01:02] NOTICE [22382] manager.c: 192.0.2.1 failed to authenticate as 'admin'
+
+# Check AMI logs
+# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.4" }
+[2016-05-06 07:08:09] NOTICE[31554] manager.c: 192.0.2.4 tried to authenticate with nonexistent user 'opennms'
+# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.5" }
+[2016-05-06 07:08:09] NOTICE[6772] manager.c: 192.0.2.5 failed to authenticate as 'Admin'
+
+# PJSip Errors
+# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.6" }
+[2016-05-06 07:08:09] NOTICE[17103] res_pjsip/pjsip_distributor.c: Request from '"test1" <sip:test1@2.3.4.5>' failed for '192.0.2.6:5678' (callid: deadbeef) - No matching endpoint found
+
+# # FreePBX Warnings
+# #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.4" }
+# [2016-05-06 07:08:09] WARNING[6410][C-00000bac] Ext. 50048943556071: Friendly Scanner from 192.0.2.4
+# #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.5" }
+# [2016-05-06 07:08:09] WARNING[6410][C-00000bac] Ext. s: Friendly Scanner from 192.0.2.5
+# #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.6" }
+# [2016-05-06 07:08:09] WARNING[6410][C-00000bac] Ext. +012345: Friendly Scanner from 192.0.2.6
+# # Yes, this does have quotes around it.

From f4551d02c95f942ee22fd778854e79f8a83db8ae Mon Sep 17 00:00:00 2001
From: Bigard Florian <florian.bigard@gmail.com>
Date: Tue, 25 Jul 2017 13:44:29 +0200
Subject: [PATCH 453/623] Fix empty logfile.log in xarf login attack action

Fix empty 3rd MIME part which contains the attack evidence (logfile.log).
---
 config/action.d/xarf-login-attack.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 3ab73817..a6cac018 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -48,9 +48,9 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(di
             PORT=<port>
             DATE=`LC_ALL=C date --date=@<time> +"%%a, %%d %%h %%Y %%T %%z"`
             if [ ! -z "$ADDRESSES" ]; then
-                (printf -- %%b "<header>\n<message>\n<report>\n";
+                (printf -- %%b "<header>\n<message>\n<report>\n\n";
                  date '+Note: Local timezone is %%z (%%Z)';
-                 printf -- %%b "<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> ${ADDRESSES//,/\" \"}
+                 printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> ${ADDRESSES//,/\" \"}
             fi
 
 actionunban =

From daf57547c66d076f31ba640de08e092fa8ca8162 Mon Sep 17 00:00:00 2001
From: Marcel Waldvogel <marcel.waldvogel@uni-konstanz.de>
Date: Sat, 29 Jul 2017 19:58:06 +0200
Subject: [PATCH 454/623] Parse ejabberd 17.06 output

E.g.:
2017-07-29 08:24:04.773 [info] <0.6668.0>@ejabberd_c2s:handle_auth_failure:433 (http_bind|ejabberd_bosh) Failed c2s PLAIN authentication for test@example.ch from ::FFFF:192.0.2.3: Invalid username or password
---
 config/filter.d/ejabberd-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf
index de27c1cd..052eea48 100644
--- a/config/filter.d/ejabberd-auth.conf
+++ b/config/filter.d/ejabberd-auth.conf
@@ -17,7 +17,7 @@
 # Values:  TEXT
 #
 failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\) Failed authentication for \S+ from (?:IP )?<HOST>(?: \({{(?:\d+,){3}\d+},\d+}\))?$
-            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:\w+:\d+ \([^\)]+\) Failed authentication for \S+ from (?:IP )?<HOST>$
+            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:\w+:\d+ \([^\)]+\) Failed (?:c2s \w+ )?authentication for \S+ from (?:IP )?(?:::FFFF:)?<HOST>(: .*)?$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

From ebd1e2c969a7568939626e3372e394a74e5a06ae Mon Sep 17 00:00:00 2001
From: Marcel Waldvogel <marcel.waldvogel@uni-konstanz.de>
Date: Sat, 29 Jul 2017 20:05:25 +0200
Subject: [PATCH 455/623] Add testcase

---
 fail2ban/tests/files/logs/ejabberd-auth | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/ejabberd-auth b/fail2ban/tests/files/logs/ejabberd-auth
index cc04883f..9f03e6a4 100644
--- a/fail2ban/tests/files/logs/ejabberd-auth
+++ b/fail2ban/tests/files/logs/ejabberd-auth
@@ -13,4 +13,8 @@ I(<0.1440.0>:ejabberd_c2s:813) : ({socket_state,tls,{tlssock,#Port<0.6910>,#Port
 # new format:
 
 # failJSON: { "time": "2015-03-19T13:57:35", "match": true , "host": "192.0.2.6" }
-2015-03-19 13:57:35.805 [info] <0.585.0>@ejabberd_c2s:wait_for_sasl_response:965 ({socket_state,p1_tls,{tlssock,#Port<0.6434>,#Port<0.6436>},<0.584.0>}) Failed authentication for robin@example.com from 192.0.2.6
\ No newline at end of file
+2015-03-19 13:57:35.805 [info] <0.585.0>@ejabberd_c2s:wait_for_sasl_response:965 ({socket_state,p1_tls,{tlssock,#Port<0.6434>,#Port<0.6436>},<0.584.0>}) Failed authentication for robin@example.com from 192.0.2.6
+
+# 17.06 "new" format:
+# failJSON: { "time": "2017-07-29T08:24:04", "match": true , "host": "192.0.2.3" }
+2017-07-29 08:24:04.773 [info] <0.6668.0>@ejabberd_c2s:handle_auth_failure:433 (http_bind|ejabberd_bosh) Failed c2s PLAIN authentication for test@example.ch from ::FFFF:192.0.2.3: Invalid username or password

From 0ef5b7c4d48254990f07a9486b1a6af6b4e866f9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 7 Aug 2017 15:24:16 +0200
Subject: [PATCH 456/623] small amend to gh-1850: removed greedy catch-all at
 end.

---
 config/filter.d/ejabberd-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/ejabberd-auth.conf b/config/filter.d/ejabberd-auth.conf
index 052eea48..48e82df5 100644
--- a/config/filter.d/ejabberd-auth.conf
+++ b/config/filter.d/ejabberd-auth.conf
@@ -17,7 +17,7 @@
 # Values:  TEXT
 #
 failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\) Failed authentication for \S+ from (?:IP )?<HOST>(?: \({{(?:\d+,){3}\d+},\d+}\))?$
-            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:\w+:\d+ \([^\)]+\) Failed (?:c2s \w+ )?authentication for \S+ from (?:IP )?(?:::FFFF:)?<HOST>(: .*)?$
+            ^(?:\.\d+)? \[info\] <0\.\d+\.\d>@ejabberd_c2s:\w+:\d+ \([^\)]+\) Failed (?:c2s \w+ )?authentication for \S+ from (?:IP )?(?:::FFFF:)?<HOST>(?:: |$)
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

From 5c538fb658c4faad29ee9c6bdae04477f455e902 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 7 Aug 2017 17:50:55 +0200
Subject: [PATCH 457/623] Recognize "unknown user" for additional auth-methods
 (pam, passwd-file, ldap, sql, etc); simplifying regular expressions (put
 "unknown user" and "invalid credentials" together as one regex).

---
 config/filter.d/dovecot.conf      | 3 +--
 fail2ban/tests/files/logs/dovecot | 3 +++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index 68c78c7a..d4b49576 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -12,8 +12,7 @@ _daemon = (auth|dovecot(-auth)?|auth-worker)
 failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
             ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
             ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
-            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
-            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$
+            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:Info: )?[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot
index 987f89f1..1614ff8c 100644
--- a/fail2ban/tests/files/logs/dovecot
+++ b/fail2ban/tests/files/logs/dovecot
@@ -40,6 +40,9 @@ Jan 29 05:13:40 mail dovecot: auth-worker(31326): pam(username,1.2.3.4): unknown
 # failJSON: { "time": "2005-01-29T05:13:50", "match": true , "host": "1.2.3.4" }
 Jan 29 05:13:50 mail dovecot: auth: passwd-file(username,1.2.3.4): unknown user
 
+# failJSON: { "time": "2005-01-29T13:54:06", "match": true , "host": "192.0.2.5" }
+Jan 29 13:54:06 auth-worker(22401): Info: sql(admin@example.de,192.0.2.5,<n4JLdHNVngZGpV2j>): unknown user
+
 # failJSON: { "time": "2005-04-19T05:22:20", "match": true , "host": "80.255.3.104" }
 Apr 19 05:22:20 vm5 auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=informix rhost=80.255.3.104
 

From 9a42ce12f4fb7ca55593d0a48af4f2f070bf261e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 12:55:13 +0200
Subject: [PATCH 458/623] amend to gh-1792: introduced new fail2ban-regex
 option "--timezone" ("--TZ"), to set time-zone used by convert of time
 format.

---
 fail2ban/client/fail2banregex.py        | 4 ++++
 fail2ban/tests/fail2banregextestcase.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 45dbfe95..89225ffb 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -127,6 +127,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   help="set alternate config directory"),
 		Option("-d", "--datepattern",
 			   help="set custom pattern used to match date/times"),
+		Option("--timezone", "--TZ", action='store', default=None,
+			   help="set time-zone used by convert time format"),
 		Option("-e", "--encoding", default=PREFER_ENC,
 			   help="File encoding. Default: system locale"),
 		Option("-r", "--raw", action='store_true', default=False,
@@ -243,6 +245,8 @@ class Fail2banRegex(object):
 			self._maxlines = 20
 		if opts.journalmatch is not None:
 			self.setJournalMatch(shlex.split(opts.journalmatch))
+		if opts.timezone:
+			self._filter.setLogTimeZone(opts.timezone)
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
 		if opts.usedns:
diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py
index 8bfedad1..f3a51773 100644
--- a/fail2ban/tests/fail2banregextestcase.py
+++ b/fail2ban/tests/fail2banregextestcase.py
@@ -195,6 +195,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 	def testVerbose(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
 			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
+			"--timezone", "UTC+0200",
 			"--verbose", "--verbose-date", "--print-no-missed",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00

From 8f76a052ece9b484997ed5c2d24981fe56681001 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 12:59:45 +0200
Subject: [PATCH 459/623] rebuild man's and MANIFEST

---
 MANIFEST                 | 14 +++++++++-----
 man/fail2ban-client.1    | 18 ++++++++++++++++--
 man/fail2ban-regex.1     | 24 +++++++++++++++---------
 man/fail2ban-server.1    | 10 ++++++++--
 man/fail2ban-testcases.1 |  9 ++++++---
 5 files changed, 54 insertions(+), 21 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index 73436511..a6fdae5a 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -3,6 +3,7 @@ bin/fail2ban-regex
 bin/fail2ban-server
 bin/fail2ban-testcases
 ChangeLog
+config/action.d/abuseipdb.conf
 config/action.d/apf.conf
 config/action.d/badips.conf
 config/action.d/badips.py
@@ -19,6 +20,7 @@ config/action.d/firewallcmd-multiport.conf
 config/action.d/firewallcmd-new.conf
 config/action.d/firewallcmd-rich-logging.conf
 config/action.d/firewallcmd-rich-rules.conf
+config/action.d/helpers-common.conf
 config/action.d/hostsdeny.conf
 config/action.d/ipfilter.conf
 config/action.d/ipfw.conf
@@ -38,6 +40,7 @@ config/action.d/mail-whois-common.conf
 config/action.d/mail-whois.conf
 config/action.d/mail-whois-lines.conf
 config/action.d/mynetwatchman.conf
+config/action.d/netscaler.conf
 config/action.d/nftables-allports.conf
 config/action.d/nftables-common.conf
 config/action.d/nftables-multiport.conf
@@ -84,6 +87,7 @@ config/filter.d/courier-auth.conf
 config/filter.d/courier-smtp.conf
 config/filter.d/cyrus-imap.conf
 config/filter.d/directadmin.conf
+config/filter.d/domino-smtp.conf
 config/filter.d/dovecot.conf
 config/filter.d/dropbear.conf
 config/filter.d/drupal-auth.conf
@@ -119,8 +123,6 @@ config/filter.d/perdition.conf
 config/filter.d/php-url-fopen.conf
 config/filter.d/portsentry.conf
 config/filter.d/postfix.conf
-config/filter.d/postfix-rbl.conf
-config/filter.d/postfix-sasl.conf
 config/filter.d/proftpd.conf
 config/filter.d/pure-ftpd.conf
 config/filter.d/qmail.conf
@@ -216,11 +218,11 @@ fail2ban/tests/clientbeautifiertestcase.py
 fail2ban/tests/clientreadertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
-fail2ban/tests/config/filter.d/common.conf
 fail2ban/tests/config/filter.d/simple.conf
 fail2ban/tests/config/filter.d/test.conf
 fail2ban/tests/config/filter.d/test.local
 fail2ban/tests/config/filter.d/zzz-generic-example.conf
+fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
 fail2ban/tests/config/jail.conf
 fail2ban/tests/databasetestcase.py
 fail2ban/tests/datedetectortestcase.py
@@ -276,6 +278,7 @@ fail2ban/tests/files/logs/courier-auth
 fail2ban/tests/files/logs/courier-smtp
 fail2ban/tests/files/logs/cyrus-imap
 fail2ban/tests/files/logs/directadmin
+fail2ban/tests/files/logs/domino-smtp
 fail2ban/tests/files/logs/dovecot
 fail2ban/tests/files/logs/dropbear
 fail2ban/tests/files/logs/drupal-auth
@@ -309,8 +312,6 @@ fail2ban/tests/files/logs/perdition
 fail2ban/tests/files/logs/php-url-fopen
 fail2ban/tests/files/logs/portsentry
 fail2ban/tests/files/logs/postfix
-fail2ban/tests/files/logs/postfix-rbl
-fail2ban/tests/files/logs/postfix-sasl
 fail2ban/tests/files/logs/proftpd
 fail2ban/tests/files/logs/pure-ftpd
 fail2ban/tests/files/logs/qmail
@@ -336,6 +337,8 @@ fail2ban/tests/files/logs/webmin-auth
 fail2ban/tests/files/logs/wuftpd
 fail2ban/tests/files/logs/xinetd-fail
 fail2ban/tests/files/logs/zzz-generic-example
+fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline
+fail2ban/tests/files/testcase01a.log
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log
@@ -344,6 +347,7 @@ fail2ban/tests/files/testcase-journal.log
 fail2ban/tests/files/testcase-multiline.log
 fail2ban/tests/files/testcase-usedns.log
 fail2ban/tests/files/testcase-wrong-char.log
+fail2ban/tests/files/zzz-sshd-obsolete-multiline.log
 fail2ban/tests/filtertestcase.py
 fail2ban/tests/__init__.py
 fail2ban/tests/misctestcase.py
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 281d8f5b..08de0af4 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,5 +1,5 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
-.TH FAIL2BAN-CLIENT "1" "September 2016" "fail2ban-client v0.10.0a2" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
+.TH FAIL2BAN-CLIENT "1" "August 2017" "fail2ban-client v0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
@@ -29,6 +29,9 @@ logging level
 \fB\-d\fR
 dump configuration. For debugging
 .TP
+\fB\-t\fR, \fB\-\-test\fR
+test configuration (can be also specified with start parameters)
+.TP
 \fB\-i\fR
 interactive mode
 .TP
@@ -53,6 +56,9 @@ start server in async mode (for internal usage only, don't read configuration)
 \fB\-\-timeout\fR
 timeout to wait for the server (for internal usage only, don't read configuration)
 .TP
+\fB\-\-str2sec\fR <STRING>
+convert time abbreviation format to seconds
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 display this help message
 .TP
@@ -188,6 +194,10 @@ JAIL CONFIGURATION
 \fBset <JAIL> idle on|off\fR
 sets the idle state of <JAIL>
 .TP
+\fBset <JAIL> ignoreself true|false\fR
+allows the ignoring of own IP
+addresses
+.TP
 \fBset <JAIL> addignoreip <IP>\fR
 adds <IP> to the ignore list of
 <JAIL>
@@ -332,6 +342,10 @@ for <JAIL>
 gets the journal filter match for
 <JAIL>
 .TP
+\fBget <JAIL> ignoreself\fR
+gets the current value of the
+ignoring the own IP addresses
+.TP
 \fBget <JAIL> ignoreip\fR
 gets the list of ignored IP
 addresses for <JAIL>
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 3f31a376..94122aa4 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,5 +1,5 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
-.TH FAIL2BAN-REGEX "1" "September 2016" "fail2ban-regex 0.10.0a2" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
+.TH FAIL2BAN-REGEX "1" "August 2017" "fail2ban-regex 0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
@@ -27,9 +27,6 @@ a string representing a 'failregex'
 .TP
 filename
 path to a filter file (filter.d/sshd.conf)
-.TP
-filtername[option=value, ..., option=value]
-short path to a filter relative filter.d in configuration base (sshd[mode=aggressive])
 .SS "IGNOREREGEX:"
 .TP
 string
@@ -45,12 +42,15 @@ show program's version number and exit
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-c\fR CONFIGBASE, \fB\-\-config\fR=\fI\,CONFIGBASE\/\fR
-set alternate config base directory (default /etc/fail2ban)
+\fB\-c\fR CONFIG, \fB\-\-config\fR=\fI\,CONFIG\/\fR
+set alternate config directory
 .TP
 \fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR
 set custom pattern used to match date/times
 .TP
+\fB\-\-timezone\fR=\fI\,TIMEZONE\/\fR, \fB\-\-TZ\fR=\fI\,TIMEZONE\/\fR
+set time\-zone used by convert time format
+.TP
 \fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fI\,ENCODING\/\fR
 File encoding. Default: system locale
 .TP
@@ -63,7 +63,7 @@ DNS specified replacement of tags <HOST> in regexp
 addresses only)
 .TP
 \fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fI\,MAXLINES\/\fR
-maxlines for multi\-line regex
+maxlines for multi\-line regex.
 .TP
 \fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fI\,JOURNALMATCH\/\fR
 journalctl style matches overriding filter file.
@@ -73,7 +73,13 @@ journalctl style matches overriding filter file.
 Log level for the Fail2Ban logger to use
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
-Be verbose in output
+Increase verbosity
+.TP
+\fB\-\-verbosity\fR=\fI\,VERBOSE\/\fR
+Set numerical level of verbosity (0..4)
+.TP
+\fB\-\-verbose\-date\fR, \fB\-\-VD\fR
+Verbose date patterns/regex in output
 .TP
 \fB\-D\fR, \fB\-\-debuggex\fR
 Produce debuggex.com urls for debugging there
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 09dcd65a..711ecd6f 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,5 +1,5 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
-.TH FAIL2BAN-SERVER "1" "September 2016" "fail2ban-server v0.10.0a2" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
+.TH FAIL2BAN-SERVER "1" "August 2017" "fail2ban-server v0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
@@ -32,6 +32,9 @@ Note. If fail2ban running as systemd-service, for logging to the systemd-journal
 \fB\-d\fR
 dump configuration. For debugging
 .TP
+\fB\-t\fR, \fB\-\-test\fR
+test configuration (can be also specified with start parameters)
+.TP
 \fB\-i\fR
 interactive mode
 .TP
@@ -56,6 +59,9 @@ start server in async mode (for internal usage only, don't read configuration)
 \fB\-\-timeout\fR
 timeout to wait for the server (for internal usage only, don't read configuration)
 .TP
+\fB\-\-str2sec\fR <STRING>
+convert time abbreviation format to seconds
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 display this help message
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index 26dbae96..0348d246 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,5 +1,5 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.3.
-.TH FAIL2BAN-TESTCASES "1" "September 2016" "fail2ban-testcases 0.10.0a2" "User Commands"
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
+.TH FAIL2BAN-TESTCASES "1" "August 2017" "fail2ban-testcases 0.10.0a2" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS
@@ -18,7 +18,10 @@ show this help message and exit
 \fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR
 Log level for the logger to use during running tests
 .TP
-\fB\-v\fR VERBOSITY, \fB\-\-verbosity\fR=\fI\,VERBOSITY\/\fR
+\fB\-v\fR
+Increase verbosity
+.TP
+\fB\-\-verbosity\fR=\fI\,VERBOSITY\/\fR
 Set numerical level of verbosity (0..4)
 .TP
 \fB\-\-log\-direct\fR

From 51c54b3253a54aec4043b898826cdb3dfcac5fb8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 13:04:36 +0200
Subject: [PATCH 460/623] ChangeLog entry for
 9a42ce12f4fb7ca55593d0a48af4f2f070bf261e

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index d7e77ff3..32f7b521 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -120,6 +120,7 @@ TODO: implementing of options resp. other tasks from PR #1346
 * Add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
 * Implemented zone abbreviations (like CET, CEST, etc.) and abbr+-offset functionality (accept zones
   like 'CET+0100'), for the list of abbreviations see strptime.TZ_STR;
+* Introduced new option `--timezone` (resp. `--TZ`) for `fail2ban-regex`.
 * Tokens `%z` and `%Z` are changed (more precise now);
 * Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations and/or offset-based
   zones (implemented as enhancement using custom `datepattern`, because may be too dangerous for default

From 5ce8d4f741768c59fef5216323e9937ffb44f885 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 13 Apr 2017 12:11:17 +0200
Subject: [PATCH 461/623] fixes default backend handling (as default used value
 of `known/backend`, which can now be overridden in default section of
 jail.local); introduces fallback for `known/option`: interpolate missing
 `known/option` as `option` from default section

---
 config/paths-common.conf           |  2 +-
 fail2ban/client/configparserinc.py | 56 ++++++++++++++++++++++++++----
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index 9072136c..522957fe 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -7,7 +7,7 @@ after  = paths-overrides.local
 
 [DEFAULT]
 
-default_backend = auto
+default_backend = %(known/backend)s
 
 sshd_log = %(syslog_authpriv)s
 sshd_backend = %(default_backend)s
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 6de513cd..7d63682a 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -32,8 +32,8 @@ from ..helpers import getLogger
 if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
-	from configparser import ConfigParser as SafeConfigParser, NoSectionError, \
-		BasicInterpolation
+	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
+		InterpolationMissingOptionError, NoSectionError
 
 	# And interpolation of __name__ was simply removed, thus we need to
 	# decorate default interpolator to handle it
@@ -52,20 +52,43 @@ if sys.version_info >= (3,2):
 		But should be fine to reincarnate for our use case
 		"""
 		def _interpolate_some(self, parser, option, accum, rest, section, map,
-							  depth):
+							  *args, **kwargs):
 			if section and not (__name__ in map):
 				map = map.copy()		  # just to be safe
 				map['__name__'] = section
-			return super(BasicInterpolationWithName, self)._interpolate_some(
-				parser, option, accum, rest, section, map, depth)
+			try:
+				return super(BasicInterpolationWithName, self)._interpolate_some(
+					parser, option, accum, rest, section, map, *args, **kwargs)
+			except InterpolationMissingOptionError as e:
+				# fallback: try to wrap missing default options as "known/options":
+				if not parser._map_known_defaults(section, option, rest, map): # pragma: no cover
+					raise e
+				# try again:
+				return super(BasicInterpolationWithName, self)._interpolate_some(
+					parser, option, accum, rest, section, map, *args, **kwargs)
 
 else: # pragma: no cover
-	from ConfigParser import SafeConfigParser, NoSectionError
+	from ConfigParser import SafeConfigParser, \
+		InterpolationMissingOptionError, NoSectionError
+
+	# Interpolate missing known/option as option from default section
+	SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some
+	def _interpolate_some(self, option, accum, rest, section, map, *args, **kwargs):
+		try:
+			return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
+		except InterpolationMissingOptionError as e:
+			# fallback: try to wrap missing default options as "known/options":
+			if self._map_known_defaults(section, option, rest, map): # pragma: no cover
+				raise e
+			# try again:
+			return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
+	SafeConfigParser._interpolate_some = _interpolate_some
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 logLevel = 7
 
+
 __all__ = ['SafeConfigParserWithIncludes']
 
 
@@ -100,6 +123,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	_KNOWN_OPTSUBST_CRE = re.compile(r'%\(known/([^\)]+)\)s')
+
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
 
 	if sys.version_info >= (3,2):
@@ -117,6 +142,25 @@ after = 1.conf
 			SafeConfigParser.__init__(self, *args, **kwargs)
 			self._cfg_share = share_config
 
+	def _map_known_defaults(self, section, option, rest, map):
+		""" Fallback: try to wrap missing default options as "known/options"
+		"""
+		known = SafeConfigParserWithIncludes._KNOWN_OPTSUBST_CRE.findall(rest)
+		if not known: # pragma: no cover
+			return 0
+		for opt in known:
+			kopt = 'known/'+opt
+			if kopt not in map:
+				try:
+					v = self._defaults[opt]
+				except KeyError:
+					continue
+				self._defaults[kopt] = v
+				try: # for python 2.6 we should duplicate it in map-vars also:
+					map[kopt] = v
+				except: pass
+		return 1
+
 	@property
 	def share_config(self):
 		return self._cfg_share

From e52f483557f5be7416a9fa4bb4c5f71bf9ea4272 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 13 Apr 2017 15:20:35 +0200
Subject: [PATCH 462/623] Config reader's: introduced new syntax
 `%(section/option)s`, in opposite to extended interpolation of python 3
 `${section:option}` work with all supported python version in fail2ban and
 this syntax is like our another features like `%(known/option)s`, etc.;
 Variable `default_backend` switched to `%(default/backend)s`, so totally
 backwards compatible now, but now the setting of parameter `backend` in
 default section of `jail.local` can overwrite default backend also. Test
 cases extended: test targeted section options "section/option" (default and
 cross sections options);

---
 config/paths-common.conf               |   2 +-
 fail2ban/client/configparserinc.py     | 108 ++++++++++++++++---------
 fail2ban/client/configreader.py        |   2 +-
 fail2ban/tests/clientreadertestcase.py |  27 +++++++
 4 files changed, 97 insertions(+), 42 deletions(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index 522957fe..51323d6b 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -7,7 +7,7 @@ after  = paths-overrides.local
 
 [DEFAULT]
 
-default_backend = %(known/backend)s
+default_backend = %(default/backend)s
 
 sshd_log = %(syslog_authpriv)s
 sshd_backend = %(default_backend)s
diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 7d63682a..1ac980e0 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -56,14 +56,8 @@ if sys.version_info >= (3,2):
 			if section and not (__name__ in map):
 				map = map.copy()		  # just to be safe
 				map['__name__'] = section
-			try:
-				return super(BasicInterpolationWithName, self)._interpolate_some(
-					parser, option, accum, rest, section, map, *args, **kwargs)
-			except InterpolationMissingOptionError as e:
-				# fallback: try to wrap missing default options as "known/options":
-				if not parser._map_known_defaults(section, option, rest, map): # pragma: no cover
-					raise e
-				# try again:
+				# try to wrap section options like %(section/option)s:
+				parser._map_section_options(section, option, rest, map)
 				return super(BasicInterpolationWithName, self)._interpolate_some(
 					parser, option, accum, rest, section, map, *args, **kwargs)
 
@@ -74,16 +68,16 @@ else: # pragma: no cover
 	# Interpolate missing known/option as option from default section
 	SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some
 	def _interpolate_some(self, option, accum, rest, section, map, *args, **kwargs):
-		try:
-			return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
-		except InterpolationMissingOptionError as e:
-			# fallback: try to wrap missing default options as "known/options":
-			if self._map_known_defaults(section, option, rest, map): # pragma: no cover
-				raise e
-			# try again:
-			return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
+		# try to wrap section options like %(section/option)s:
+		self._map_section_options(section, option, rest, map)
+		return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
 	SafeConfigParser._interpolate_some = _interpolate_some
 
+try:
+	from configparser import _UNSET
+except ImportError:
+	_UNSET = object()
+
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 logLevel = 7
@@ -123,7 +117,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
-	_KNOWN_OPTSUBST_CRE = re.compile(r'%\(known/([^\)]+)\)s')
+	SECTION_OPT_CRE = re.compile(r'^(\w+)/(.+)$')
+	SECTION_OPTSUBST_CRE = re.compile(r'%\((\w+/([^\)]+))\)s')
 
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
 
@@ -142,22 +137,43 @@ after = 1.conf
 			SafeConfigParser.__init__(self, *args, **kwargs)
 			self._cfg_share = share_config
 
-	def _map_known_defaults(self, section, option, rest, map):
-		""" Fallback: try to wrap missing default options as "known/options"
+	def _map_section_options(self, section, option, rest, map):
 		"""
-		known = SafeConfigParserWithIncludes._KNOWN_OPTSUBST_CRE.findall(rest)
-		if not known: # pragma: no cover
+		Interpolates values of the section options (name syntax `%(section/option)s`).
+
+		Fallback: try to wrap missing default options as "default/options" resp. "known/options"
+		"""
+		if '/' not in rest or '%(' not in rest: # pragma: no cover
 			return 0
-		for opt in known:
-			kopt = 'known/'+opt
-			if kopt not in map:
-				try:
-					v = self._defaults[opt]
-				except KeyError:
-					continue
-				self._defaults[kopt] = v
-				try: # for python 2.6 we should duplicate it in map-vars also:
-					map[kopt] = v
+		soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest)
+		if not soptrep: # pragma: no cover
+			return 0
+		for sopt, opt in soptrep:
+			if sopt not in map:
+				sec = sopt[:~len(opt)]
+				seclwr = sec.lower()
+				if seclwr != 'default':
+					if seclwr == 'known':
+						# try get raw value from known options:
+						try:
+							v = self._sections['KNOWN'][opt]
+						except KeyError:
+							# fallback to default:
+							try:
+								v = self._defaults[opt]
+							except KeyError:
+								continue
+					else:
+						# get substituted value of opt in section:
+						v = self.get(sec, opt)
+				else:
+					try:
+						v = self._defaults[opt]
+					except KeyError:
+						continue
+				self._defaults[sopt] = v
+				try: # for some python versions need to duplicate it in map-vars also:
+					map[sopt] = v
 				except: pass
 		return 1
 
@@ -244,6 +260,18 @@ after = 1.conf
 	def get_sections(self):
 		return self._sections
 
+	def get(self, sec, opt, raw=False, vars={}, fallback=_UNSET):
+		try:
+			return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars)
+		except:
+			sopt = SafeConfigParserWithIncludes.SECTION_OPT_CRE.match(opt)
+			if not sopt: raise
+			sec, opt = sopt.groups()
+			if sec.lower() == 'default':
+				# get default raw value:
+				return self._defaults[opt]
+			return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars)
+
 	def options(self, section, withDefault=True):
 		"""Return a list of option names for the given section name.
 
@@ -303,11 +331,7 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							sk = {}
-							for k, v in s2.iteritems():
-								if not k.startswith('known/') and k != '__name__':
-									sk['known/'+k] = v
-							s2.update(sk)
+							self.merge_section('KNOWN', s2, '')
 							# merge section
 							s2.update(s)
 						else:
@@ -324,14 +348,18 @@ after = 1.conf
 		else:
 			return SafeConfigParser.read(self, fileNamesFull)
 
-	def merge_section(self, section, options, pref='known/'):
+	def merge_section(self, section, options, pref=None):
 		alls = self.get_sections()
-		if pref == '':
-			alls[section].update(options)
+		try:
+			sec = alls[section]
+		except KeyError:
+			alls[section] = sec = dict()
+		if not pref:
+			sec.update(options)
 			return
 		sk = {}
 		for k, v in options.iteritems():
 			if not k.startswith(pref) and k != '__name__':
 				sk[pref+k] = v
-		alls[section].update(sk)
+		sec.update(sk)
 
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index bbc18384..381af759 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -110,7 +110,7 @@ class ConfigReader():
 
 	def sections(self):
 		try:
-			return self._cfg.sections()
+			return (n for n in self._cfg.sections() if n != 'KNOWN')
 		except AttributeError:
 			return []
 
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 6cc2f659..325a1d68 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -162,6 +162,33 @@ c = d ;in line comment
 		self.assertEqual(self.c.get('DEFAULT', 'b'), 'a')
 		self.assertEqual(self.c.get('DEFAULT', 'c'), 'd')
 
+	def testTargetedSectionOptions(self):
+		self.assertFalse(self.c.read('g'))	# nothing is there yet
+		self._write("g.conf", value=None, content="""
+[DEFAULT]
+a = def-a
+b = def-b,a:`%(a)s`
+c = def-c,b:"%(b)s"
+d = def-d-b:"%(known/b)s"
+[jail]
+a = jail-a-%(test/a)s
+b = jail-b-%(test/b)s
+[test]
+a = test-a-%(default/a)s
+b = test-b-%(known/b)s
+""")
+		self.assertTrue(self.c.read('g'))
+		self.assertEqual(self.c.get('test', 'a'), 'test-a-def-a')
+		self.assertEqual(self.c.get('test', 'b'), 'test-b-def-b,a:`test-a-def-a`')
+		self.assertEqual(self.c.get('jail', 'a'), 'jail-a-test-a-def-a')
+		self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`test-a-def-a`')
+		self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"')
+		self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('test', 'd'),  'def-d-b:"def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"')
+		self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"')
+
 
 class JailReaderTest(LogCaptureTestCase):
 

From 0782b28460057a913ac7ef7bf2fcf6bdc2debc58 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 21:13:49 +0200
Subject: [PATCH 463/623] allow usage of "-" character in the name of sections
 by extended interpolation (like `%(my-section/some-value)s`).

---
 fail2ban/client/configparserinc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 1ac980e0..cc245b0a 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -117,8 +117,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
-	SECTION_OPT_CRE = re.compile(r'^(\w+)/(.+)$')
-	SECTION_OPTSUBST_CRE = re.compile(r'%\((\w+/([^\)]+))\)s')
+	SECTION_OPT_CRE = re.compile(r'^([\w\-]+)/(.+)$')
+	SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s')
 
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
 

From 9a768c19d0b3f7039584d640de9fd1322072517b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 21:48:50 +0200
Subject: [PATCH 464/623] Fixes raw to interpolated value by recursive
 substitution. Avoid too deeply recursive extended interpolation cross
 sections (test covered now);

---
 fail2ban/client/configparserinc.py     |  4 ++--
 fail2ban/tests/clientreadertestcase.py | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index cc245b0a..7cc9b9fa 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -164,8 +164,8 @@ after = 1.conf
 							except KeyError:
 								continue
 					else:
-						# get substituted value of opt in section:
-						v = self.get(sec, opt)
+						# get raw value of opt in section:
+						v = self.get(sec, opt, raw=True)
 				else:
 					try:
 						v = self._defaults[opt]
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 325a1d68..c3a10c36 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -170,24 +170,31 @@ a = def-a
 b = def-b,a:`%(a)s`
 c = def-c,b:"%(b)s"
 d = def-d-b:"%(known/b)s"
+
 [jail]
 a = jail-a-%(test/a)s
 b = jail-b-%(test/b)s
+y = %(test/y)s
+
 [test]
 a = test-a-%(default/a)s
 b = test-b-%(known/b)s
+x = %(test/x)s
+y = %(jail/y)s
 """)
 		self.assertTrue(self.c.read('g'))
 		self.assertEqual(self.c.get('test', 'a'), 'test-a-def-a')
 		self.assertEqual(self.c.get('test', 'b'), 'test-b-def-b,a:`test-a-def-a`')
 		self.assertEqual(self.c.get('jail', 'a'), 'jail-a-test-a-def-a')
-		self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`test-a-def-a`')
-		self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('jail', 'b'), 'jail-b-test-b-def-b,a:`jail-a-test-a-def-a`')
+		self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"')
 		self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"')
 		self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"')
 		self.assertEqual(self.c.get('test', 'd'),  'def-d-b:"def-b,a:`test-a-def-a`"')
 		self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"')
 		self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"')
+		self.assertRaises(Exception, self.c.get, 'test', 'x')
+		self.assertRaises(Exception, self.c.get, 'jail', 'y')
 
 
 class JailReaderTest(LogCaptureTestCase):

From 6f4fde2c29743286907fec5c5afc5fb11122401f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 8 Aug 2017 21:50:38 +0200
Subject: [PATCH 465/623] Update changelog and man/jail.conf.5

---
 ChangeLog       | 7 +++++++
 man/jail.conf.5 | 9 +++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 32f7b521..2672889a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -134,6 +134,13 @@ TODO: implementing of options resp. other tasks from PR #1346
         Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone 
 	abbreviations.
 * `filter.d/courier-auth.conf`: support failed logins with method only
+* Config reader's: introduced new syntax `%(section/option)s`, in opposite to extended interpolation of
+  python 3 `${section:option}` work with all supported python version in fail2ban and this syntax is 
+  like our another features like `%(known/option)s`, etc. (gh-1750)
+* Variable `default_backend` switched to `%(default/backend)s`, so totally backwards compatible now,
+  but now the setting of parameter `backend` in default section of `jail.local` can overwrite default
+  backend also (see gh-1750). In the future versions parameter `default_backend` can be removed (incompatibility, 
+  possibly some distributions affected).
 
 
 ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index a8afe53c..e939771b 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -90,11 +90,16 @@ indicates that the specified file is to be parsed after the current file.
 .RE
 
 Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s.
-Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g.
+
+Fail2ban has more advanced syntax (similar python extended interpolation). This extended interpolation is using \fB%(section/parameter)s\fR to denote a value from a foreign section.
+.br
+Besides cross section interpolation the value of parameter in \fI[DEFAULT]\fR section can be retrieved with \fB%(default/parameter)s\fR.
+.br
+Fail2ban supports also another feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g.
 
 .RS
 .nf
-baduseragents = IE|wget
+baduseragents = IE|wget|%(my-settings/baduseragents)s
 failregex = %(known/failregex)s
             useragent=%(baduseragents)s
 .fi

From 7549f44dedb886767eccc19a84061f9d1abd841d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 9 Aug 2017 15:36:36 +0200
Subject: [PATCH 466/623] code review: removed unused (unneeded) code, coverage
 increase...

---
 fail2ban/client/configparserinc.py | 24 +++---------------------
 1 file changed, 3 insertions(+), 21 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index 7cc9b9fa..b626be9b 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -73,11 +73,6 @@ else: # pragma: no cover
 		return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
 	SafeConfigParser._interpolate_some = _interpolate_some
 
-try:
-	from configparser import _UNSET
-except ImportError:
-	_UNSET = object()
-
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 logLevel = 7
@@ -117,7 +112,6 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
-	SECTION_OPT_CRE = re.compile(r'^([\w\-]+)/(.+)$')
 	SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s')
 
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
@@ -161,7 +155,7 @@ after = 1.conf
 							# fallback to default:
 							try:
 								v = self._defaults[opt]
-							except KeyError:
+							except KeyError: # pragma: no cover
 								continue
 					else:
 						# get raw value of opt in section:
@@ -169,7 +163,7 @@ after = 1.conf
 				else:
 					try:
 						v = self._defaults[opt]
-					except KeyError:
+					except KeyError: # pragma: no cover
 						continue
 				self._defaults[sopt] = v
 				try: # for some python versions need to duplicate it in map-vars also:
@@ -260,18 +254,6 @@ after = 1.conf
 	def get_sections(self):
 		return self._sections
 
-	def get(self, sec, opt, raw=False, vars={}, fallback=_UNSET):
-		try:
-			return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars)
-		except:
-			sopt = SafeConfigParserWithIncludes.SECTION_OPT_CRE.match(opt)
-			if not sopt: raise
-			sec, opt = sopt.groups()
-			if sec.lower() == 'default':
-				# get default raw value:
-				return self._defaults[opt]
-			return SafeConfigParser.get(self, sec, opt, raw=raw, vars=vars)
-
 	def options(self, section, withDefault=True):
 		"""Return a list of option names for the given section name.
 
@@ -279,7 +261,7 @@ after = 1.conf
 		"""
 		try:
 			opts = self._sections[section]
-		except KeyError:
+		except KeyError: # pragma: no cover
 			raise NoSectionError(section)
 		if withDefault:
 			# mix it with defaults:

From 84f552881c6ac6101b73cf213e56b230f5216c63 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@cora.nwra.com>
Date: Mon, 3 Oct 2016 16:26:11 -0600
Subject: [PATCH 467/623] Add sendmail journalmatch options

---
 config/filter.d/sendmail-auth.conf   | 4 ++++
 config/filter.d/sendmail-reject.conf | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf
index 138fbb85..7886e60d 100644
--- a/config/filter.d/sendmail-auth.conf
+++ b/config/filter.d/sendmail-auth.conf
@@ -13,6 +13,10 @@ failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]( \(may be forged\))?: po
 
 ignoreregex =
 
+[Init]
+
+journalmatch = _SYSTEMD_UNIT=sendmail.service
+
 # DEV Notes:
 #
 # Author: Daniel Black
diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf
index 2f8fd882..e408a18b 100644
--- a/config/filter.d/sendmail-reject.conf
+++ b/config/filter.d/sendmail-reject.conf
@@ -33,6 +33,8 @@ ignoreregex =
 
 [Init]
 
+journalmatch = _SYSTEMD_UNIT=sendmail.service
+
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 10
 

From 2ed8a38eca27710fc952ee4489955a863d67f23b Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 9 Aug 2017 16:27:53 +0200
Subject: [PATCH 468/623] Update cloudflare.conf

Switch to API v1 to API v4 per default
---
 config/action.d/cloudflare.conf | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf
index 15b0e31e..89df5b9e 100644
--- a/config/action.d/cloudflare.conf
+++ b/config/action.d/cloudflare.conf
@@ -41,11 +41,11 @@ actioncheck =
 # Values:  CMD
 #
 # API v1
-actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+#actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-#actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-#            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
-#            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
+actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+            -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \
+            https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -56,11 +56,11 @@ actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=
 # Values:  CMD
 #
 # API v1
-actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
+#actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
-#actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-#              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
-#              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)
+actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+              https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \
+              'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6)
 
 [Init]
 

From c5402178447289c8e0f6f7ee9c8a92b0e307d07d Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 9 Aug 2017 16:34:37 +0200
Subject: [PATCH 469/623] Update ChangeLog

action.d/cloudflare.conf - Cloudflare API v4 implementation (gh-1651)
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 99bcb4b0..5aa24b51 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,6 +28,7 @@ releases.
 ### New Features
 
 ### Enhancements
+* action.d/cloudflare.conf - Cloudflare API v4 implementation (gh-1651)
 * filter.d/kerio.conf - filter extended with new rules (see gh-1455)
 
 

From c60784540c5307d16cdc136ace5b395961492e73 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 9 Aug 2017 16:53:05 +0200
Subject: [PATCH 470/623] version bump: release 0.10.0

---
 ChangeLog                | 2 +-
 README.md                | 2 +-
 fail2ban/version.py      | 2 +-
 man/fail2ban-client.1    | 4 ++--
 man/fail2ban-regex.1     | 2 +-
 man/fail2ban-server.1    | 4 ++--
 man/fail2ban-testcases.1 | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3cbe26dc..20f3d1b5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,7 +6,7 @@
 Fail2Ban: Changelog
 ===================
 
-ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining
+ver. 0.10.0 (2017/08/09) - long-awaited 0.10th version
 -----------
 
 TODO: implementing of options resp. other tasks from PR #1346
diff --git a/README.md b/README.md
index 72c48378..e0b16ae3 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.0                 2016/??/??
+                       v0.10.0                 2017/08/09
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
diff --git a/fail2ban/version.py b/fail2ban/version.py
index 48d2ed66..1225ae5f 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.0a2"
+version = "0.10.0"
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 08de0af4..2215d70e 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "August 2017" "fail2ban-client v0.10.0a2" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "August 2017" "fail2ban-client v0.10.0" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.10.0a2 reads log file that contains password failure report
+Fail2Ban v0.10.0 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 94122aa4..c414bec3 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "August 2017" "fail2ban-regex 0.10.0a2" "User Commands"
+.TH FAIL2BAN-REGEX "1" "August 2017" "fail2ban-regex 0.10.0" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 711ecd6f..5151821e 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "August 2017" "fail2ban-server v0.10.0a2" "User Commands"
+.TH FAIL2BAN-SERVER "1" "August 2017" "fail2ban-server v0.10.0" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.10.0a2 reads log file that contains password failure report
+Fail2Ban v0.10.0 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index 0348d246..e4566005 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "August 2017" "fail2ban-testcases 0.10.0a2" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "August 2017" "fail2ban-testcases 0.10.0" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS

From 28076618fd815b8e945818da5eb9c059a04ba29b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 9 Aug 2017 17:37:40 +0200
Subject: [PATCH 471/623] back to development edition: README.md, ChangeLog,
 version.py

---
 ChangeLog           | 10 ++++++++++
 README.md           |  2 +-
 fail2ban/version.py |  2 +-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 20f3d1b5..d84ba34d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,16 @@
 Fail2Ban: Changelog
 ===================
 
+ver. 0.10.1-dev-1 (2016/??/??) - development edition
+-----------
+
+### Fixes
+
+### New Features
+
+### Enhancements
+
+
 ver. 0.10.0 (2017/08/09) - long-awaited 0.10th version
 -----------
 
diff --git a/README.md b/README.md
index e0b16ae3..bb87b935 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.0                 2017/08/09
+                       v0.10.1                 2016/??/??
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
diff --git a/fail2ban/version.py b/fail2ban/version.py
index 1225ae5f..724a0b9a 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.0"
+version = "0.10.1dev1"

From d1de20dd4181c9506c29d218c20b16fc7239cf38 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 14 Aug 2017 18:29:36 +0200
Subject: [PATCH 472/623] Move some ticket-independent tag replacements from
 ActionInfo to ADD_REPL_TAGS (will be wrapped in replaceTag into calling map).
 Thus tags `<fq-hostname>` and `<sh-hostname>` could be used without ticket
 (a. g. in `actionstart` etc.). Closes gh-1859.

---
 fail2ban/server/action.py  | 17 ++++++++++++++---
 fail2ban/server/actions.py |  7 +------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index d00458ba..81ffbe2f 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -33,7 +33,7 @@ from abc import ABCMeta
 from collections import MutableMapping
 
 from .failregex import mapTag2Opt
-from .ipdns import asip
+from .ipdns import asip, DNSUtils
 from .mytime import MyTime
 from .utils import Utils
 from ..helpers import getLogger, _merge_copy_dicts, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
@@ -52,11 +52,18 @@ FCUSTAG_CRE = re.compile(r'<F-([A-Z0-9_\-]+)>'); # currently uppercase only
 
 CONDITIONAL_FAM_RE = re.compile(r"^(\w+)\?(family)=")
 
+# Special tags:
+DYN_REPL_TAGS = {
+  # System-information:
+	"fq-hostname":	lambda: str(DNSUtils.getHostname(fqdn=True)),
+	"sh-hostname":	lambda: str(DNSUtils.getHostname(fqdn=False))
+}
 # New line, space
 ADD_REPL_TAGS = {
   "br": "\n", 
   "sp": " "
 }
+ADD_REPL_TAGS.update(DYN_REPL_TAGS)
 
 
 class CallingMap(MutableMapping, object):
@@ -571,6 +578,8 @@ class CommandAction(ActionBase):
 			if csubkey is not None:
 				cache[csubkey] = subInfo
 
+		# additional replacement as calling map:
+		ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS)
 		# substitution callable, used by interpolation of each tag
 		def substVal(m):
 			tag = m.group(1)			# tagname from match
@@ -581,7 +590,7 @@ class CommandAction(ActionBase):
 				value = subInfo.get(tag)
 				if value is None:
 					# fallback (no or default replacement)
-					return ADD_REPL_TAGS.get(tag, m.group())
+					return ADD_REPL_TAGS_CM.get(tag, m.group())
 			value = str(value)		# assure string
 			if tag in cls._escapedTags:
 				# That one needs to be escaped since its content is
@@ -651,6 +660,8 @@ class CommandAction(ActionBase):
 			# replacement for tag:
 			return value
 
+		# additional replacement as calling map:
+		ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS)
 		# substitution callable, used by interpolation of each tag
 		def substVal(m):
 			tag = m.group(1)			# tagname from match
@@ -658,7 +669,7 @@ class CommandAction(ActionBase):
 				value = aInfo[tag]
 			except KeyError:
 				# fallback (no or default replacement)
-				return ADD_REPL_TAGS.get(tag, m.group())
+				return ADD_REPL_TAGS_CM.get(tag, m.group())
 			value = str(value)		# assure string
 			# replacement for tag:
 			return escapeVal(tag, value)
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index f8f8d4e1..73042434 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -35,7 +35,6 @@ except ImportError:
 	OrderedDict = dict
 
 from .banmanager import BanManager
-from .ipdns import DNSUtils
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
@@ -306,10 +305,7 @@ class Actions(JailThread, Mapping):
 			"ipmatches":			lambda self: "\n".join(self._mi4ip(True).getMatches()),
 			"ipjailmatches":	lambda self: "\n".join(self._mi4ip().getMatches()),
 			"ipfailures":			lambda self: self._mi4ip(True).getAttempt(),
-			"ipjailfailures":	lambda self: self._mi4ip().getAttempt(),
-			# system-information:
-			"fq-hostname":	lambda self: DNSUtils.getHostname(fqdn=True),
-			"sh-hostname":	lambda self: DNSUtils.getHostname(fqdn=False)
+			"ipjailfailures":	lambda self: self._mi4ip().getAttempt()
 		}
 
 		__slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip')
@@ -368,7 +364,6 @@ class Actions(JailThread, Mapping):
 
 
 	def __getActionInfo(self, ticket):
-		ip = ticket.getIP()
 		aInfo = Actions.ActionInfo(ticket, self._jail)
 		return aInfo
 

From 786722814682a7279314505ad2a1602e52e997c8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 10:45:37 +0200
Subject: [PATCH 473/623] closes part of gh-1865: fixed "Retrieving own IPs of
 localhost failed: inet_pton() argument 2 must be string, not int" some
 python-versions resp. host configurations causes returning of integer
 (instead of ip-string)

---
 fail2ban/server/ipdns.py         | 8 ++++++--
 fail2ban/tests/filtertestcase.py | 4 ++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 6ef36888..7110d974 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -69,10 +69,14 @@ class DNSUtils:
 		for fam, ipfam in ((socket.AF_INET, IPAddr.FAM_IPv4), (socket.AF_INET6, IPAddr.FAM_IPv6)):
 			try:
 				for result in socket.getaddrinfo(dns, None, fam, 0, socket.IPPROTO_TCP):
-					ip = IPAddr(result[4][0], ipfam)
+					# if getaddrinfo returns something unexpected:
+					if len(result) < 4 or not len(result[4]): continue
+					# get ip from `(2, 1, 6, '', ('127.0.0.1', 0))`,be sure we've an ip-string
+					# (some python-versions resp. host configurations causes returning of integer there):
+					ip = IPAddr(str(result[4][0]), ipfam)
 					if ip.isValid:
 						ips.append(ip)
-			except socket.error as e:
+			except Exception as e:
 				saveerr = e
 		if not ips and saveerr:
 			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, saveerr)
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index cb0edb06..fe90f2ee 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1852,6 +1852,10 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))
 		self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips))
 
+	def testIPAddr_wrongDNS_IP(self):
+		DNSUtils.dnsToIp('`this`.dns-is-wrong.`wrong-nic`-dummy')
+		DNSUtils.ipToName('*')
+
 	def testIPAddr_Cached(self):
 		ips = [DNSUtils.dnsToIp('example.com'), DNSUtils.dnsToIp('example.com')]
 		for ip1, ip2 in zip(ips, ips):

From e5169d8f8421b3736c4610ad84c3953596b6d4b3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 11:37:22 +0200
Subject: [PATCH 474/623] pyinotify: be sure possible IOError/OSError by remove
 monitor (log-rotate? normally not raises) are handled properly.

---
 fail2ban/server/filterpyinotify.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 41b8be7c..10f51847 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -231,6 +231,9 @@ class FilterPyinotify(FileFilter):
 				return True
 		except KeyError: # pragma: no cover
 			pass
+			# EnvironmentError is parent of IOError, OSError, etc.
+		except EnvironmentError as e: # pragma: no cover (normally unreached)
+			logSys.error("Remove file monitor for %s causes %s", path, e)
 		return False
 
 	def _addDirWatcher(self, path_dir):
@@ -249,6 +252,9 @@ class FilterPyinotify(FileFilter):
 			self.__monitor.rm_watch(wdInt)
 		except KeyError: # pragma: no cover
 			pass
+			# EnvironmentError is parent of IOError, OSError, etc.
+		except EnvironmentError as e: # pragma: no cover (normally unreached)
+			logSys.error("Remove monitor for the parent directory %s causes %s", path_dir, e)
 		logSys.debug("Removed monitor for the parent directory %s", path_dir)
 
 	##

From 08646bc3399f62d6a7eed87579f72614fd3e9355 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 13:14:42 +0200
Subject: [PATCH 475/623] Always supply jail name as name parameter (if not
 specified explicit in the action parameters `action[name=...]`). Avoid usage
 of the same chains (etc.) if someone use `action` instead of `banaction` jail
 parameter.

---
 fail2ban/client/actionreader.py        |  3 +++
 fail2ban/tests/clientreadertestcase.py | 15 +++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index ace0b898..559705bc 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -54,6 +54,9 @@ class ActionReader(DefinitionInitConfigReader):
 		if actname is None:
 			actname = file_
 			initOpts["actname"] = actname
+		# always supply jail name as name parameter if not specified in options:
+		if initOpts.get("name") is None:
+			initOpts["name"] = jailName
 		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index c3a10c36..f9c5bf4c 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -33,7 +33,7 @@ from ..client import configparserinc
 from ..client.jailreader import JailReader
 from ..client.filterreader import FilterReader
 from ..client.jailsreader import JailsReader
-from ..client.actionreader import ActionReader
+from ..client.actionreader import ActionReader, CommandAction
 from ..client.configurator import Configurator
 from ..server.mytime import MyTime
 from ..version import version
@@ -571,7 +571,8 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['set', 'brokenaction', 'addaction', 'brokenaction'],
 			 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
 				 ['actionban', 'hit with big stick <ip>'],
-				 ['actname', 'brokenaction']
+				 ['actname', 'brokenaction'],
+				 ['name', 'brokenaction']
 			 ]],
 			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
 			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
@@ -612,6 +613,16 @@ class JailsReaderTest(LogCaptureTestCase):
 					# all must have some actionban defined
 					self.assertTrue(actionReader._opts.get('actionban', '').strip(),
 						msg="Action file %r is lacking actionban" % actionConfig)
+					# test name of jail is set in options (also if not supplied within parameters):
+					opts = actionReader.getCombined(
+						ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
+					self.assertEqual(opts.get('name'), 'TEST',
+						msg="Action file %r does not contains jail-name 'f2b-TEST'" % actionConfig)
+					# and the name is substituted (test several actions surely contains name-interpolation):
+					if actionName in ('pf', 'iptables-allports', 'iptables-multiport'):
+						#print('****', actionName, opts.get('actionstart', ''))
+						self.assertIn('f2b-TEST', opts.get('actionstart', ''),
+							msg="Action file %r: interpolation of actionstart does not contains jail-name 'f2b-TEST'" % actionConfig)
 				self.assertIn('Init', actionReader.sections(),
 						msg="Action file %r is lacking [Init] section" % actionConfig)
 

From 19e59fff3edecde4e7be18f8b1f1023603981457 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 15:38:44 +0200
Subject: [PATCH 476/623] ChangeLog: added incompatibility list (compared to
 v.0.9)

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d84ba34d..abcb0bb2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,31 @@
 Fail2Ban: Changelog
 ===================
 
+Incompatibility list (compared to v.0.9):
+-----------
+
+* Filter (or `failregex`) internal capture-groups:
+
+  - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should
+    rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)`
+    (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings).
+
+    Of course you can always your own capture-group (like below `_cond_ip_`) to do this.
+    ```
+    testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1"
+    fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$"
+    ```
+  - New internal groups (currently reserved for internal usage):
+    `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if
+    mapping from tag `<F-*>` used in failregex (e. g. `user` by `<F-USER>`).
+
+* v.0.10 uses more precise date template handling, that can be theoretically incompatible to some
+  user configurations resp. `datepattern`.
+
+* Since v0.10 fail2ban supports the matching of the IPv6 addresses, but not all ban actions are
+  IPv6-capable now.
+
+
 ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 

From f6ccede2f183f44f947e9ae2030a1aed1f73ba4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20K=C3=B6ppe?= <alexander@koeppe.rocks>
Date: Wed, 16 Aug 2017 11:33:45 +0200
Subject: [PATCH 477/623] Update pf.conf fixing #1863

Fix #1863
Introduce own PF anchors for fail2ban rules.
---
 config/action.d/pf.conf | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index deb38c09..02fbcd8d 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -15,8 +15,8 @@
 #
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
-actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -f-
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -a f2b/<name> -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -a f2b/<name> -f-
 
 # Option:  start_on_demand - to start action on demand
 # Example: `action=pf[actionstart_on_demand=true]`
@@ -27,16 +27,16 @@ actionstart_on_demand = false
 # Values:  CMD
 #
 # we only disable PF rules we've installed prior
-actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
-             pfctl -t <tablename>-<name> -T flush
-             pfctl -t <tablename>-<name> -T kill
+actionstop = pfctl -a f2b/<name> -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -a f2b/<name> -f-
+             pfctl -a f2b/<name> -t <tablename>-<name> -T flush
+             pfctl -a f2b/<name> -t <tablename>-<name> -T kill
 
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = pfctl -sr | grep -q <tablename>-<name>
+actioncheck = pfctl -a f2b/<name> -sr | grep -q <tablename>-<name>
 
 
 # Option:  actionban
@@ -47,7 +47,7 @@ actioncheck = pfctl -sr | grep -q <tablename>-<name>
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = pfctl -t <tablename>-<name> -T add <ip>
+actionban = pfctl -a f2b/<name> -t <tablename>-<name> -T add <ip>
 
 
 # Option:  actionunban
@@ -59,7 +59,7 @@ actionban = pfctl -t <tablename>-<name> -T add <ip>
 # Values:  CMD
 #
 # note -r option used to remove matching rule
-actionunban = pfctl -t <tablename>-<name> -T delete <ip>
+actionunban = pfctl -a f2b/<name> -t <tablename>-<name> -T delete <ip>
 
 [Init]
 # Option:  tablename

From 33874d6e53952248b8a6471514afea898b033672 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 11:58:39 +0200
Subject: [PATCH 478/623] action.d/pf.conf: anchored call arguments combined as
 `<pfctl>` parameter; test cases fixed;

---
 config/action.d/pf.conf          | 23 +++++++----
 fail2ban/tests/servertestcase.py | 66 ++++++++++++++++----------------
 2 files changed, 48 insertions(+), 41 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 02fbcd8d..943f016a 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -15,8 +15,8 @@
 #
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
-actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -a f2b/<name> -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | pfctl -a f2b/<name> -f-
+actionstart = echo "table <<tablename>-<name>> persist counters" | <pfctl> -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | <pfctl> -f-
 
 # Option:  start_on_demand - to start action on demand
 # Example: `action=pf[actionstart_on_demand=true]`
@@ -27,16 +27,16 @@ actionstart_on_demand = false
 # Values:  CMD
 #
 # we only disable PF rules we've installed prior
-actionstop = pfctl -a f2b/<name> -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -a f2b/<name> -f-
-             pfctl -a f2b/<name> -t <tablename>-<name> -T flush
-             pfctl -a f2b/<name> -t <tablename>-<name> -T kill
+actionstop = <pfctl> -sr 2>/dev/null | grep -v <tablename>-<name> | <pfctl> -f-
+             <pfctl> -t <tablename>-<name> -T flush
+             <pfctl> -t <tablename>-<name> -T kill
 
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = pfctl -a f2b/<name> -sr | grep -q <tablename>-<name>
+actioncheck = <pfctl> -sr | grep -q <tablename>-<name>
 
 
 # Option:  actionban
@@ -47,7 +47,7 @@ actioncheck = pfctl -a f2b/<name> -sr | grep -q <tablename>-<name>
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = pfctl -a f2b/<name> -t <tablename>-<name> -T add <ip>
+actionban = <pfctl> -t <tablename>-<name> -T add <ip>
 
 
 # Option:  actionunban
@@ -59,7 +59,14 @@ actionban = pfctl -a f2b/<name> -t <tablename>-<name> -T add <ip>
 # Values:  CMD
 #
 # note -r option used to remove matching rule
-actionunban = pfctl -a f2b/<name> -t <tablename>-<name> -T delete <ip>
+actionunban = <pfctl> -t <tablename>-<name> -T delete <ip>
+
+# Option: pfctl
+#
+# Use anchor as jailname to manipulate affected rulesets only.
+# If more parameter expected it can be extended with `pf[pfctl="<known/pfctl> ..."]`
+# 
+pfctl = pfctl -a f2b/<name>
 
 [Init]
 # Option:  tablename
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 68b9951c..9fb5ecf0 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1503,59 +1503,59 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				('j-w-pf', 'pf[name=%(__name__)s, actionstart_on_demand=false]', {
 					'ip4': (), 'ip6': (),
 					'start': (
-						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -f-`',
+						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -a f2b/j-w-pf -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -a f2b/j-w-pf -f-`',
 					),
 					'stop': (
-						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
-						'`pfctl -t f2b-j-w-pf -T flush`',
-						'`pfctl -t f2b-j-w-pf -T kill`',
+						'`pfctl -a f2b/j-w-pf -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -a f2b/j-w-pf -f-`',
+						'`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T flush`',
+						'`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T kill`',
 					),
-					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
-					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
-					'ip4-ban':   ("`pfctl -t f2b-j-w-pf -T add 192.0.2.1`",),
-					'ip4-unban': ("`pfctl -t f2b-j-w-pf -T delete 192.0.2.1`",),
-					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
-					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
+					'ip4-check': ("`pfctl -a f2b/j-w-pf -sr | grep -q f2b-j-w-pf`",),
+					'ip6-check': ("`pfctl -a f2b/j-w-pf -sr | grep -q f2b-j-w-pf`",),
+					'ip4-ban':   ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
 				# pf multiport with custom port --
 				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
 					'ip4': (), 'ip6': (),
 					'start': (
-						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -f-`',
+						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -a f2b/j-w-pf-mp -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -a f2b/j-w-pf-mp -f-`',
 					),
 					'stop': (
-						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -f-`',
-						'`pfctl -t f2b-j-w-pf-mp -T flush`',
-						'`pfctl -t f2b-j-w-pf-mp -T kill`',
+						'`pfctl -a f2b/j-w-pf-mp -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -a f2b/j-w-pf-mp -f-`',
+						'`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T flush`',
+						'`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T kill`',
 					),
-					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
-					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-mp`",),
-					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 192.0.2.1`",),
-					'ip4-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 192.0.2.1`",),
-					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-mp -T add 2001:db8::`",),
-					'ip6-unban': ("`pfctl -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
+					'ip4-check': ("`pfctl -a f2b/j-w-pf-mp -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip6-check': ("`pfctl -a f2b/j-w-pf-mp -sr | grep -q f2b-j-w-pf-mp`",),
+					'ip4-ban':   ("`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T delete 2001:db8::`",),
 				}),
 				# pf allports -- test additionally "actionstart_on_demand" was set to true
 				('j-w-pf-ap', 'pf[actiontype=<allports>, actionstart_on_demand=true][name=%(__name__)s]', {
 					'ip4': (), 'ip6': (),
 					'ip4-start': (
-						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -f-`',
+						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -a f2b/j-w-pf-ap -f-`',
+						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -a f2b/j-w-pf-ap -f-`',
 					),
 					'ip6-start': (), # the same as ipv4
 					'stop': (
-						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -f-`',
-						'`pfctl -t f2b-j-w-pf-ap -T flush`',
-						'`pfctl -t f2b-j-w-pf-ap -T kill`',
+						'`pfctl -a f2b/j-w-pf-ap -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -a f2b/j-w-pf-ap -f-`',
+						'`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T flush`',
+						'`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T kill`',
 					),
-					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
-					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf-ap`",),
-					'ip4-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 192.0.2.1`",),
-					'ip4-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 192.0.2.1`",),
-					'ip6-ban':   ("`pfctl -t f2b-j-w-pf-ap -T add 2001:db8::`",),
-					'ip6-unban': ("`pfctl -t f2b-j-w-pf-ap -T delete 2001:db8::`",),
+					'ip4-check': ("`pfctl -a f2b/j-w-pf-ap -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip6-check': ("`pfctl -a f2b/j-w-pf-ap -sr | grep -q f2b-j-w-pf-ap`",),
+					'ip4-ban':   ("`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T delete 2001:db8::`",),
 				}),
 				# firewallcmd-multiport --
 				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {

From 10c0d954017fac270bf1c568e4b02e94d5949b58 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 20:48:18 +0200
Subject: [PATCH 479/623] prevent sporadic assert errors on nested lists/dict
 differ at some level (mostly causes on python 3.4 because of different dict
 hashing); introduces new test assertion routine `asesertSortedEqual` for
 comparison regardless elements order (regarding level/nestedOnly arguments).

---
 fail2ban/tests/banmanagertestcase.py   |  8 ++---
 fail2ban/tests/clientreadertestcase.py | 18 +++++-----
 fail2ban/tests/databasetestcase.py     |  6 ++--
 fail2ban/tests/filtertestcase.py       | 14 ++++----
 fail2ban/tests/misctestcase.py         | 19 ++++++++++
 fail2ban/tests/servertestcase.py       | 20 +++++------
 fail2ban/tests/utils.py                | 49 +++++++++++++++++++++++++-
 7 files changed, 100 insertions(+), 34 deletions(-)

diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index 33cd2dac..e3d0f620 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -203,7 +203,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		ticket = BanTicket("8.0.0.0", 1167606000.0)
 		self.assertTrue(self.__banManager.addBanTicket(ticket))
 		cymru_info = self._getBanListExtendedCymruInfo()
-		self.assertDictEqual(dict((k, sorted(v)) for k, v in cymru_info.iteritems()),
-						  {"asn": sorted(["nxdomain", "3356",]),
-						   "country": sorted(["nxdomain", "US"]),
-						   "rir": sorted(["nxdomain", "arin"])})
+		self.assertSortedEqual(cymru_info,
+						  {"asn": ["nxdomain", "3356",],
+						   "country": ["nxdomain", "US"],
+						   "rir": ["nxdomain", "arin"]})
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index f9c5bf4c..4b2ec0e5 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -409,7 +409,7 @@ class FilterReaderTest(unittest.TestCase):
 
 		# Add sort as configreader uses dictionary and therefore order
 		# is unreliable
-		self.assertEqual(sorted(filterReader.convert()), sorted(output))
+		self.assertSortedEqual(filterReader.convert(), output)
 
 		filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"},
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
@@ -417,7 +417,7 @@ class FilterReaderTest(unittest.TestCase):
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
 		output[-1][-1] = "5"
-		self.assertEqual(sorted(filterReader.convert()), sorted(output))
+		self.assertSortedEqual(filterReader.convert(), output)
 
 	def testFilterReaderSubstitionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
@@ -426,7 +426,7 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
-		self.assertEqual(sorted(c), sorted(output))
+		self.assertSortedEqual(c, output)
 
 	def testFilterReaderSubstitionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
@@ -435,7 +435,7 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
-		self.assertEqual(sorted(c), sorted(output))
+		self.assertSortedEqual(c, output)
 
 	def testFilterReaderSubstitionKnown(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>']]
@@ -446,7 +446,7 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.read()
 		filterReader.getOptions(None)
 		c = filterReader.convert()
-		self.assertEqual(sorted(c), sorted(output))
+		self.assertSortedEqual(c, output)
 
 	def testFilterReaderSubstitionFail(self):
 		# directly subst the same var :
@@ -555,8 +555,8 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertRaises(ValueError, jails.convert)
 		comm_commands = jails.convert(allow_no_files=True)
 		self.maxDiff = None
-		self.assertEqual(sorted(comm_commands),
-			sorted([['add', 'emptyaction', 'auto'],
+		self.assertSortedEqual(comm_commands,
+			[['add', 'emptyaction', 'auto'],
 			 ['add', 'test-known-interp', 'auto'],
 			 ['multi-set', 'test-known-interp', 'addfailregex', [
 			   'failure test 1 (filter.d/test.conf) <HOST>',
@@ -592,7 +592,7 @@ class JailsReaderTest(LogCaptureTestCase):
 				"Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"],
 			 ['config-error',
 				"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
-			 ]))
+			 ])
 		self.assertLogged("Errors in jail 'missingbitsjail'.")
 		self.assertNotLogged("Skipping...")
 		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
@@ -801,7 +801,7 @@ class JailsReaderTest(LogCaptureTestCase):
 
 			# and there is logging information left to be passed into the
 			# server
-			self.assertEqual(sorted(commands),
+			self.assertSortedEqual(commands,
 							 [['set', 'dbfile',
 								'/var/lib/fail2ban/fail2ban.sqlite3'],
 							  ['set', 'dbpurgeage', '1d'],
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index cbfc1517..33fc4413 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -336,9 +336,9 @@ class DatabaseTest(LogCaptureTestCase):
 
 		tickets = self.db.getBansMerged()
 		self.assertEqual(len(tickets), 2)
-		self.assertEqual(
-			sorted(list(set(ticket.getIP() for ticket in tickets))),
-			sorted([ticket.getIP() for ticket in tickets]))
+		self.assertSortedEqual(
+			list(set(ticket.getIP() for ticket in tickets)),
+			[ticket.getIP() for ticket in tickets])
 
 		tickets = self.db.getBansMerged(jail=jail2)
 		self.assertEqual(len(tickets), 1)
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index fe90f2ee..52187b3d 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -1286,7 +1286,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.waitForTicks(1)
 			self.waitFailTotal(6, 10)
 			self.assertTrue(Utils.wait_for(lambda: len(self.jail) == 2, 10))
-			self.assertEqual(sorted([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()]), 
+			self.assertSortedEqual([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()], 
 				["192.0.2.1", "192.0.2.2"])
 
 	cls = MonitorJournalFailures
@@ -1334,7 +1334,7 @@ class GetFailures(LogCaptureTestCase):
 		self.assertEqual(self.filter.getLogPaths(), [GetFailures.FILENAME_01])
 		self.filter.addLogPath(GetFailures.FILENAME_02, tail=True)
 		self.assertEqual(self.filter.getLogCount(), 2)
-		self.assertEqual(sorted(self.filter.getLogPaths()), sorted([GetFailures.FILENAME_01, GetFailures.FILENAME_02]))
+		self.assertSortedEqual(self.filter.getLogPaths(), [GetFailures.FILENAME_01, GetFailures.FILENAME_02])
 
 	def testTail(self):
 		# There must be no containters registered, otherwise [-1] indexing would be wrong
@@ -1547,7 +1547,7 @@ class GetFailures(LogCaptureTestCase):
 					_ticket_tuple(self.filter.failManager.toBan())[0:3])
 			except FailManagerEmpty:
 				break
-		self.assertEqual(sorted(foundList), sorted(output))
+		self.assertSortedEqual(foundList, output)
 
 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
@@ -1582,7 +1582,7 @@ class GetFailures(LogCaptureTestCase):
 					_ticket_tuple(self.filter.failManager.toBan())[0:3])
 			except FailManagerEmpty:
 				break
-		self.assertEqual(sorted(foundList), sorted(output))
+		self.assertSortedEqual(foundList, output)
 
 
 class DNSUtilsTests(unittest.TestCase):
@@ -1679,10 +1679,10 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
 		# sort ipaddr, IPv4 is always smaller as IPv6
-		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
 		# sort ipaddr, IPv4 is always smaller as IPv6
-		self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+		self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1695,7 +1695,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
 				# sort ipaddr, IPv4 is always smaller as IPv6
-				self.assertEqual(sorted(res), ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
+				self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 		# pure ips:
diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index cd841a0f..77e65d7c 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -285,6 +285,25 @@ class TestsUtilsTest(LogCaptureTestCase):
 		self.assertDictEqual({'A': [1, 2]}, {'A': [1, 2]})
 		self.assertRaises(AssertionError, self.assertDictEqual, 
 			{'A': [1, 2]}, {'A': [2, 1]})
+		## assertSortedEqual:
+		self.assertSortedEqual(['A', 'B'], ['B', 'A'])
+		self.assertSortedEqual([['A', 'B']], [['B', 'A']], level=2)
+		self.assertSortedEqual([['A', 'B']], [['B', 'A']], nestedOnly=False)
+		self.assertRaises(AssertionError, lambda: self.assertSortedEqual(
+			[['A', 'B']], [['B', 'A']], level=1, nestedOnly=True))
+		self.assertSortedEqual({'A': ['A', 'B']}, {'A': ['B', 'A']}, nestedOnly=False)
+		self.assertRaises(AssertionError, lambda: self.assertSortedEqual(
+			{'A': ['A', 'B']}, {'A': ['B', 'A']}, level=1, nestedOnly=True))
+		self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'],
+			nestedOnly=False)
+		self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'],
+			level=-1)
+		self.assertRaises(AssertionError, lambda: self.assertSortedEqual(
+			['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z']))
+		self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]",
+			self.assertSortedEqual, ['A'], ['C', 'B'])
+		self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]",
+			self.assertSortedEqual, ['A', 'B'], ['C', 'B'])
 
 	def testFormatterWithTraceBack(self):
 		strout = StringIO()
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 9fb5ecf0..0b454ae4 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -124,14 +124,14 @@ class TransmitterBase(unittest.TestCase):
 			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdAdd, value])
-			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
+			self.assertSortedEqual((ret[0], map(str, ret[1])), (0, map(str, values[:n+1])), level=2)
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
+			self.assertSortedEqual((ret[0], map(str, ret[1])), (0, map(str, values[:n+1])), level=2)
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdDel, value])
-			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
+			self.assertSortedEqual((ret[0], map(str, ret[1])), (0, map(str, values[n+1:])), level=2)
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
+			self.assertSortedEqual((ret[0], map(str, ret[1])), (0, map(str, values[n+1:])), level=2)
 
 	def jailAddDelRegexTest(self, cmd, inValues, outValues, jail):
 		cmdAdd = "add" + cmd
@@ -690,9 +690,9 @@ class Transmitter(TransmitterBase):
 					"be skipped" % (sys.version))
 				return
 			raise
-		self.assertEqual(
-			sorted(self.transm.proceed(["get", self.jailName,
-				"actionproperties", action])[1]),
+		self.assertSortedEqual(
+			self.transm.proceed(["get", self.jailName,
+				"actionproperties", action])[1],
 			['opt1', 'opt2'])
 		self.assertEqual(
 			self.transm.proceed(["get", self.jailName, "action", action,
@@ -702,9 +702,9 @@ class Transmitter(TransmitterBase):
 			self.transm.proceed(["get", self.jailName, "action", action,
 				"opt2"]),
 			(0, None))
-		self.assertEqual(
-			sorted(self.transm.proceed(["get", self.jailName, "actionmethods",
-				action])[1]),
+		self.assertSortedEqual(
+			self.transm.proceed(["get", self.jailName, "actionmethods",
+				action])[1],
 			['ban', 'start', 'stop', 'testmethod', 'unban'])
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "action", action,
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 9f92c26b..5f837573 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -481,8 +481,8 @@ def gatherTests(regexps=None, opts=None):
 # Forwards compatibility of unittest.TestCase for some early python versions
 #
 
+import difflib, pprint
 if not hasattr(unittest.TestCase, 'assertDictEqual'):
-	import difflib, pprint
 	def assertDictEqual(self, d1, d2, msg=None):
 		self.assert_(isinstance(d1, dict), 'First argument is not a dictionary')
 		self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary')
@@ -495,6 +495,53 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'):
 			self.fail(msg)
 	unittest.TestCase.assertDictEqual = assertDictEqual
 
+def assertSortedEqual(self, a, b, level=1, nestedOnly=True, msg=None):
+	"""Compare complex elements (like dict, list or tuple) in sorted order until
+	level 0 not reached (initial level = -1 meant all levels),
+	or if nestedOnly set to True and some of the objects still contains nested lists or dicts.
+	"""
+	# used to recognize having element as nested dict, list or tuple:
+	def _is_nested(v):
+		if isinstance(v, dict):
+			return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues())
+		return any(isinstance(v, (dict, list, tuple)) for v in v)
+	# level comparison routine:
+	def _assertSortedEqual(a, b, level, nestedOnly):
+		# first the lengths:
+		if len(a) != len(b):
+			raise ValueError('%r != %r' % (a, b))
+		# if not allow sorting of nested - just compare directly:
+		if not level and (nestedOnly and (not _is_nested(a) and not _is_nested(b))):
+			if a == b:
+				return
+			raise ValueError('%r != %r' % (a, b))
+		if isinstance(a, dict) and isinstance(b, dict): # compare dict's:
+			for k, v1 in a.iteritems():
+				v2 = b[k]
+				if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)):
+					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly)
+				elif v1 != v2:
+					raise ValueError('%r != %r' % (a, b))
+		else: # list, tuple, something iterable:
+			a = sorted(a, key=repr)
+			b = sorted(b, key=repr)
+			for v1, v2 in zip(a, b):
+				if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)):
+					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly)
+				elif v1 != v2:
+					raise ValueError('%r != %r' % (a, b))
+	# compare and produce assertion-error by exception:
+	try:
+		_assertSortedEqual(a, b, level, nestedOnly)
+	except Exception as e:
+		standardMsg = e.args[0] if isinstance(e, ValueError) else (str(e) + "\nwithin:")
+		diff = ('\n' + '\n'.join(difflib.ndiff(
+			pprint.pformat(a).splitlines(),
+			pprint.pformat(b).splitlines())))
+		msg = msg or (standardMsg + diff)
+		self.fail(msg)
+unittest.TestCase.assertSortedEqual = assertSortedEqual
+
 if not hasattr(unittest.TestCase, 'assertRaisesRegexp'):
 	def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs):
 		try:

From 72bd666797c8285caf2123bbfdef9f23b3b96dc3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 20:50:06 +0200
Subject: [PATCH 480/623] Fixes representation of IPAddr (likely the string
 representation, enclosed in single-quotes).

---
 fail2ban/server/ipdns.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py
index 7110d974..2841eac1 100644
--- a/fail2ban/server/ipdns.py
+++ b/fail2ban/server/ipdns.py
@@ -298,7 +298,7 @@ class IPAddr(object):
 			self._family = IPAddr.CIDR_RAW
 
 	def __repr__(self):
-		return self.ntoa
+		return repr(self.ntoa)
 
 	def __str__(self):
 		return self.ntoa

From a3c6bb601d9b6dd2c447b54dfe41b05dfd81ade8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 20:50:33 +0200
Subject: [PATCH 481/623] Fixes version, causes "UserWarning: Normalizing
 '0.10.1dev1' to '0.10.1.dev1'" during setup-process.

---
 fail2ban/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/version.py b/fail2ban/version.py
index 724a0b9a..f9638bab 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.1dev1"
+version = "0.10.1.dev1"

From 69a6d0e653eea7ca61dc7aaeac133109749944a1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 16 Aug 2017 21:15:28 +0200
Subject: [PATCH 482/623] amend to 10c0d954017fac270bf1c568e4b02e94d5949b58:
 order in cymru-info can variate on each level, sorted using key=str
 (otherwise `['nxdomain', u'US'] != ['US', 'nxdomain']` may occur on some
 python versions).

---
 fail2ban/tests/banmanagertestcase.py |  2 +-
 fail2ban/tests/utils.py              | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py
index e3d0f620..0391ae3e 100644
--- a/fail2ban/tests/banmanagertestcase.py
+++ b/fail2ban/tests/banmanagertestcase.py
@@ -206,4 +206,4 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		self.assertSortedEqual(cymru_info,
 						  {"asn": ["nxdomain", "3356",],
 						   "country": ["nxdomain", "US"],
-						   "rir": ["nxdomain", "arin"]})
+						   "rir": ["nxdomain", "arin"]}, level=-1, key=str)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index 5f837573..0cf40e86 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -495,7 +495,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'):
 			self.fail(msg)
 	unittest.TestCase.assertDictEqual = assertDictEqual
 
-def assertSortedEqual(self, a, b, level=1, nestedOnly=True, msg=None):
+def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None):
 	"""Compare complex elements (like dict, list or tuple) in sorted order until
 	level 0 not reached (initial level = -1 meant all levels),
 	or if nestedOnly set to True and some of the objects still contains nested lists or dicts.
@@ -506,7 +506,7 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, msg=None):
 			return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues())
 		return any(isinstance(v, (dict, list, tuple)) for v in v)
 	# level comparison routine:
-	def _assertSortedEqual(a, b, level, nestedOnly):
+	def _assertSortedEqual(a, b, level, nestedOnly, key):
 		# first the lengths:
 		if len(a) != len(b):
 			raise ValueError('%r != %r' % (a, b))
@@ -519,20 +519,20 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, msg=None):
 			for k, v1 in a.iteritems():
 				v2 = b[k]
 				if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)):
-					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly)
+					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key)
 				elif v1 != v2:
 					raise ValueError('%r != %r' % (a, b))
 		else: # list, tuple, something iterable:
-			a = sorted(a, key=repr)
-			b = sorted(b, key=repr)
+			a = sorted(a, key=key)
+			b = sorted(b, key=key)
 			for v1, v2 in zip(a, b):
 				if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)):
-					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly)
+					_assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key)
 				elif v1 != v2:
 					raise ValueError('%r != %r' % (a, b))
 	# compare and produce assertion-error by exception:
 	try:
-		_assertSortedEqual(a, b, level, nestedOnly)
+		_assertSortedEqual(a, b, level, nestedOnly, key)
 	except Exception as e:
 		standardMsg = e.args[0] if isinstance(e, ValueError) else (str(e) + "\nwithin:")
 		diff = ('\n' + '\n'.join(difflib.ndiff(

From f84e58e769f93ce3a6491b311023018daace0d80 Mon Sep 17 00:00:00 2001
From: Jacques Distler <distler@golem.ph.utexas.edu>
Date: Thu, 17 Aug 2017 09:13:32 -0500
Subject: [PATCH 483/623] Tweaks to action.d/pf.conf Document recent changes.
 Add an option to customize the pf block rule (surely, what the user really
 wants, here, is "block quick").

---
 config/action.d/pf.conf          | 18 +++++++++++++++++-
 fail2ban/tests/servertestcase.py |  6 +++---
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index 943f016a..c70c8e43 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -15,8 +15,17 @@
 #
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+# also, these rulesets are loaded into (nested) anchors
+# to enable them, add
+#     anchor f2b {
+#        name1
+#        name2
+#        ...
+#     }
+# to your main pf ruleset, where "namei" are the names of the jails
+# which invoke this action
 actionstart = echo "table <<tablename>-<name>> persist counters" | <pfctl> -f- 
-              echo "block proto <protocol> from <<tablename>-<name>> to <actiontype>" | <pfctl> -f-
+              echo "<block> proto <protocol> from <<tablename>-<name>> to <actiontype>" | <pfctl> -f-
 
 # Option:  start_on_demand - to start action on demand
 # Example: `action=pf[actionstart_on_demand=true]`
@@ -75,6 +84,12 @@ pfctl = pfctl -a f2b/<name>
 #
 tablename = f2b
 
+# Option: block
+#
+# The action you want pf to take.
+# Probably, you want "block quick", but adjust as needed.
+block = block quick
+
 # Option:  protocol
 # Notes.:  internally used by config reader for interpolations.
 # Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
@@ -96,3 +111,4 @@ allports = any
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
 multiport = any port <port>
+
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 0b454ae4..0b888407 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1504,7 +1504,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -a f2b/j-w-pf -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -a f2b/j-w-pf -f-`',
+						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -a f2b/j-w-pf -f-`',
 					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -a f2b/j-w-pf -f-`',
@@ -1523,7 +1523,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -a f2b/j-w-pf-mp -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -a f2b/j-w-pf-mp -f-`',
+						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -a f2b/j-w-pf-mp -f-`',
 					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf-mp -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -a f2b/j-w-pf-mp -f-`',
@@ -1542,7 +1542,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'ip4-start': (
 						'`echo "table <f2b-j-w-pf-ap> persist counters" | pfctl -a f2b/j-w-pf-ap -f-`',
-						'`echo "block proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -a f2b/j-w-pf-ap -f-`',
+						'`echo "block quick proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -a f2b/j-w-pf-ap -f-`',
 					),
 					'ip6-start': (), # the same as ipv4
 					'stop': (

From 8e6b4346dc505ad4a279bb98c75ce7f2d5cf3a1d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 18 Aug 2017 13:30:50 +0200
Subject: [PATCH 484/623] avoid using "ANSI_X3.4-1968" as preferred encoding,
 if missing environment variables 'LANGUAGE', 'LC_ALL', 'LC_CTYPE', and 'LANG'
 (especially critical if default value `encoding = auto` configured).

As PoC and coverage (this case fails without this "fix"):
$ env -i PATH="$PATH" bin/fail2ban-testcases --fast --no-network testAddBanInvalidEncoded
---
 fail2ban/helpers.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 556ca173..7eaa59f8 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -34,6 +34,10 @@ from .server.mytime import MyTime
 
 
 PREFER_ENC = locale.getpreferredencoding()
+# correct prefered encoding if lang not set in environment:
+if PREFER_ENC.startswith('ANSI_'): # pragma: no cover
+	if all((os.getenv(v) in (None, "") for v in ('LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'LANG'))):
+		PREFER_ENC = 'UTF-8';
 
 
 def formatExceptionInfo():

From 3be32adefb964f2b79b00efcc31e7e7fc0e42b9f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 18 Aug 2017 14:37:29 +0200
Subject: [PATCH 485/623] Replace not posix-compliant grep option: fgrep with
 `-q` option can cause 141 exit code in some cases (see gh-1389).

---
 ChangeLog                     | 2 ++
 config/action.d/bsd-ipfw.conf | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 5aa24b51..c6f8d8e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -20,6 +20,8 @@ releases.
 * action.d/bsd-ipfw.conf
     - Make the rule number, the action starts looking for a free slot to insert
       the new rule, configurable (gh-1689)
+    - Replace not posix-compliant grep option: fgrep with `-q` option can cause 
+      141 exit code in some cases (gh-1389)
 * filter.d/apache-overflows.conf:
     - Fixes resources greedy expression (see gh-1790);
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf
index 65a5e39b..f08905b8 100644
--- a/config/action.d/bsd-ipfw.conf
+++ b/config/action.d/bsd-ipfw.conf
@@ -14,7 +14,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' &> /dev/null || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
 
 
 # Option:  actionstop

From b0e5efb631a3bab1b6a0ad5ccdba731ab57f926d Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 18 Aug 2017 15:26:09 +0200
Subject: [PATCH 486/623] bsd-ipfw.conf: sh-compliant redirect of stderr
 together with stdout

---
 config/action.d/bsd-ipfw.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf
index f08905b8..cbd6a15d 100644
--- a/config/action.d/bsd-ipfw.conf
+++ b/config/action.d/bsd-ipfw.conf
@@ -14,7 +14,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' &> /dev/null || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
 
 
 # Option:  actionstop

From 5b4bc2aafda544dcd606dc68c1706247be6992f0 Mon Sep 17 00:00:00 2001
From: Pavel Mihadyuk <p_mihadyuk@wargaming.net>
Date: Tue, 22 Aug 2017 18:19:55 +0300
Subject: [PATCH 487/623] Added filter for phpMyAdmin+syslog (>=4.7.0). Closes
 #1713

---
 ChangeLog                                       |  1 +
 config/filter.d/phpmyadmin-syslog.conf          | 17 +++++++++++++++++
 .../tests/files/logs/phpmyadmin-syslog.conf     |  2 ++
 3 files changed, 20 insertions(+)
 create mode 100644 config/filter.d/phpmyadmin-syslog.conf
 create mode 100644 fail2ban/tests/files/logs/phpmyadmin-syslog.conf

diff --git a/ChangeLog b/ChangeLog
index 5aa24b51..4b1e5e61 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,6 +30,7 @@ releases.
 ### Enhancements
 * action.d/cloudflare.conf - Cloudflare API v4 implementation (gh-1651)
 * filter.d/kerio.conf - filter extended with new rules (see gh-1455)
+* filter.d/phpmyadmin-syslog.conf - new filter for phpMyAdmin using syslog for auth logging
 
 
 ver. 0.9.7 (2017/05/11) - awaiting-victory
diff --git a/config/filter.d/phpmyadmin-syslog.conf b/config/filter.d/phpmyadmin-syslog.conf
new file mode 100644
index 00000000..79eac71a
--- /dev/null
+++ b/config/filter.d/phpmyadmin-syslog.conf
@@ -0,0 +1,17 @@
+# Fail2Ban fitler for the phpMyAdmin-syslog
+#
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+_daemon = phpMyAdmin
+
+failregex = ^%(__prefix_line)suser denied: .* \(mysql-denied\) from <HOST>\s*$
+
+ignoreregex =
+
+
+# Author: Pavel Mihadyuk
diff --git a/fail2ban/tests/files/logs/phpmyadmin-syslog.conf b/fail2ban/tests/files/logs/phpmyadmin-syslog.conf
new file mode 100644
index 00000000..eef6e3b2
--- /dev/null
+++ b/fail2ban/tests/files/logs/phpmyadmin-syslog.conf
@@ -0,0 +1,2 @@
+# failJSON: { "time": "2017-08-22T14:50:22", "match": true , "host": "81.62.21.201" }
+Aug 22 14:50:22 eurostream phpMyAdmin[16358]: user denied: root (mysql-denied) from 81.62.21.201

From 41994fcb5609bc36ea5f137b65d5e797ea27cefc Mon Sep 17 00:00:00 2001
From: Pavel Mihadyuk <p_mihadyuk@wargaming.net>
Date: Tue, 22 Aug 2017 18:46:33 +0300
Subject: [PATCH 488/623] Added filter for phpMyAdmin+syslog (>=4.7.0)

---
 fail2ban/tests/files/logs/phpmyadmin-syslog | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 fail2ban/tests/files/logs/phpmyadmin-syslog

diff --git a/fail2ban/tests/files/logs/phpmyadmin-syslog b/fail2ban/tests/files/logs/phpmyadmin-syslog
new file mode 100644
index 00000000..eef6e3b2
--- /dev/null
+++ b/fail2ban/tests/files/logs/phpmyadmin-syslog
@@ -0,0 +1,2 @@
+# failJSON: { "time": "2017-08-22T14:50:22", "match": true , "host": "81.62.21.201" }
+Aug 22 14:50:22 eurostream phpMyAdmin[16358]: user denied: root (mysql-denied) from 81.62.21.201

From d09304b897e4499fbf8095344e9818a71a2ba6fd Mon Sep 17 00:00:00 2001
From: Pavel Mihadyuk <p_mihadyuk@wargaming.net>
Date: Tue, 22 Aug 2017 19:00:48 +0300
Subject: [PATCH 489/623] phpmyadmin-syslog: added default jail config

---
 config/jail.conf | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/config/jail.conf b/config/jail.conf
index 9296b6af..cf652fe2 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -860,3 +860,9 @@ logpath = /var/log/slapd.log
 port    = smtp,ssmtp
 filter  = domino-smtp
 logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log
+
+[phpmyadmin-syslog]
+port    = http,https
+filter  = phpmyadmin-syslog
+logpath = %(syslog_authpriv)s
+backend = %(syslog_backend)s

From e3b061e94b54067525c5e7f2ac716d1c838c9f20 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 23 Aug 2017 12:55:36 +0200
Subject: [PATCH 490/623] - `files/fail2ban.service` renamed as template to
 `files/fail2ban.service.in`; - setup process generates
 `build/fail2ban.service` from `files/fail2ban.service.in` using distribution
 related bin-path; - bug-fixing by running setup with option `--dry-run`
 (note: specify option `--dry-run` before `install`, like `python setup.py
 --dry-run install`); - test cases extended to cover dry-run.

---
 fail2ban/tests/misctestcase.py                | 20 +++++++-
 .../{fail2ban.service => fail2ban.service.in} |  8 ++--
 setup.py                                      | 47 ++++++++++++++++---
 3 files changed, 63 insertions(+), 12 deletions(-)
 rename files/{fail2ban.service => fail2ban.service.in} (70%)

diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 77e65d7c..a5e8fc17 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -101,6 +101,22 @@ class SetupTest(unittest.TestCase):
 				"Seems to be running with python distribution %s"
 				" -- install can be tested only with system distribution %s" % (str(tuple(sys.version_info)), sysVer))
 
+	def testSetupInstallDryRun(self):
+		if not self.setup:
+			return			  # if verbose skip didn't work out
+		tmp = tempfile.mkdtemp()
+		# suppress stdout (and stderr) if not heavydebug
+		supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
+		try:
+			# try dry-run:
+			self.assertEqual(os.system("%s %s --dry-run install --disable-2to3 --root=%s%s"
+					  % (sys.executable, self.setup , tmp, supdbgout)), 0)
+			# check nothing was created:
+			self.assertTrue(not os.listdir(tmp))
+		finally:
+			# clean up
+			shutil.rmtree(tmp)
+
 	def testSetupInstallRoot(self):
 		if not self.setup:
 			return			  # if verbose skip didn't work out
@@ -108,8 +124,8 @@ class SetupTest(unittest.TestCase):
 		# suppress stdout (and stderr) if not heavydebug
 		supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
-			os.system("%s %s install --disable-2to3 --dry-run --root=%s%s"
-					  % (sys.executable, self.setup, tmp, supdbgout))
+			self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s"
+					  % (sys.executable, self.setup, tmp, supdbgout)), 0)
 
 			def strippath(l):
 				return [x[len(tmp)+1:] for x in l]
diff --git a/files/fail2ban.service b/files/fail2ban.service.in
similarity index 70%
rename from files/fail2ban.service
rename to files/fail2ban.service.in
index 6eeba957..7114a938 100644
--- a/files/fail2ban.service
+++ b/files/fail2ban.service.in
@@ -7,11 +7,11 @@ PartOf=iptables.service firewalld.service
 [Service]
 Type=simple
 ExecStartPre=/bin/mkdir -p /var/run/fail2ban
-ExecStart=/usr/bin/fail2ban-server -xf start
+ExecStart=@BINDIR@/fail2ban-server -xf start
 # if should be logged in systemd journal, use following line or set logtarget to stdout in fail2ban.local
-# ExecStart=/usr/bin/fail2ban-server -xf --logtarget=stdout start
-ExecStop=/usr/bin/fail2ban-client stop
-ExecReload=/usr/bin/fail2ban-client reload
+# ExecStart=@BINDIR@/fail2ban-server -xf --logtarget=stdout start
+ExecStop=@BINDIR@/fail2ban-client stop
+ExecReload=@BINDIR@/fail2ban-client reload
 PIDFile=/var/run/fail2ban/fail2ban.pid
 Restart=on-failure
 RestartPreventExitStatus=0 255
diff --git a/setup.py b/setup.py
index 5754db49..11748778 100755
--- a/setup.py
+++ b/setup.py
@@ -50,6 +50,7 @@ except ImportError:
 
 import os
 from os.path import isfile, join, isdir, realpath
+import re
 import sys
 import warnings
 from glob import glob
@@ -57,11 +58,25 @@ from glob import glob
 from fail2ban.setup import updatePyExec
 
 
+source_dir = os.path.realpath(os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__
+))
+
 # Wrapper to install python binding (to current python version):
 class install_scripts_f2b(install_scripts):
 
 	def get_outputs(self):
 		outputs = install_scripts.get_outputs(self)
+		# setup.py --dry-run install:
+		dry_run = not outputs
+		self.update_scripts(dry_run)
+		if dry_run:
+			#bindir = self.install_dir
+			bindir = self.build_dir
+			print('creating fail2ban-python binding -> %s (dry-run, real path can be different)' % (bindir,))
+			print('Copying content of %s to %s' % (self.build_dir, self.install_dir));
+			return outputs
 		fn = None
 		for fn in outputs:
 			if os.path.basename(fn) == 'fail2ban-server':
@@ -71,6 +86,27 @@ class install_scripts_f2b(install_scripts):
 		updatePyExec(bindir)
 		return outputs
 
+	def update_scripts(self, dry_run=False):
+		buildroot = os.path.dirname(self.build_dir)
+		print('Creating %s/fail2ban.service (from fail2ban.service.in): @BINDIR@ -> %s' % (buildroot, self.install_dir))
+		with open(os.path.join(source_dir, 'files/fail2ban.service.in'), 'r') as fn:
+			lines = fn.readlines()
+		fn = None
+		if not dry_run:
+			fn = open(os.path.join(buildroot, 'fail2ban.service'), 'w')
+		try:
+			for ln in lines:
+				ln = re.sub(r'@BINDIR@', lambda v: self.install_dir, ln)
+				if dry_run:
+					sys.stdout.write(' | ' + ln)
+					continue
+				fn.write(ln)
+		finally:
+			if fn: fn.close()
+		if dry_run:
+			print(' `')
+
+
 # Wrapper to specify fail2ban own options:
 class install_command_f2b(install):
 	user_options = install.user_options + [
@@ -94,11 +130,7 @@ class install_command_f2b(install):
 
 
 # Update fail2ban-python env to current python version (where f2b-modules located/installed)
-rootdir = os.path.realpath(os.path.dirname(
-	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
-	sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__
-))
-updatePyExec(os.path.join(rootdir, 'bin'))
+updatePyExec(os.path.join(source_dir, 'bin'))
 
 if setuptools and "test" in sys.argv:
 	import logging
@@ -270,5 +302,8 @@ if isdir("/usr/lib/fail2ban"):
 if sys.argv[1] == "install":
 	print("")
 	print("Please do not forget to update your configuration files.")
-	print("They are in /etc/fail2ban/.")
+	print("They are in \"/etc/fail2ban/\".")
+	print("")
+	print("You can also install systemd service-unit file from \"build/fail2ban.service\"")
+	print("resp. corresponding init script from \"files/*-initd\".")
 	print("")

From f451cf34b350c1280747c22fb869905acbd892b6 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 23 Aug 2017 13:20:51 +0200
Subject: [PATCH 491/623] don't check return code by dry-run: returns 256 on
 some python/setuptool versions.

---
 fail2ban/tests/misctestcase.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index a5e8fc17..373a7178 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -109,8 +109,8 @@ class SetupTest(unittest.TestCase):
 		supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
 			# try dry-run:
-			self.assertEqual(os.system("%s %s --dry-run install --disable-2to3 --root=%s%s"
-					  % (sys.executable, self.setup , tmp, supdbgout)), 0)
+			os.system("%s %s --dry-run install --disable-2to3 --root=%s%s"
+					  % (sys.executable, self.setup , tmp, supdbgout))
 			# check nothing was created:
 			self.assertTrue(not os.listdir(tmp))
 		finally:

From 4c1abe1cbf08005f20aae01eea735feeed251a8e Mon Sep 17 00:00:00 2001
From: Pavel Mihadyuk <p_mihadyuk@wargaming.net>
Date: Wed, 23 Aug 2017 16:55:16 +0300
Subject: [PATCH 492/623] phpmyadmin-syslog: removed excess file, fixed test,
 updated failregex

---
 config/filter.d/phpmyadmin-syslog.conf           | 3 ++-
 fail2ban/tests/files/logs/phpmyadmin-syslog      | 4 ++--
 fail2ban/tests/files/logs/phpmyadmin-syslog.conf | 2 --
 3 files changed, 4 insertions(+), 5 deletions(-)
 delete mode 100644 fail2ban/tests/files/logs/phpmyadmin-syslog.conf

diff --git a/config/filter.d/phpmyadmin-syslog.conf b/config/filter.d/phpmyadmin-syslog.conf
index 79eac71a..5b0862bb 100644
--- a/config/filter.d/phpmyadmin-syslog.conf
+++ b/config/filter.d/phpmyadmin-syslog.conf
@@ -9,9 +9,10 @@ before = common.conf
 
 _daemon = phpMyAdmin
 
-failregex = ^%(__prefix_line)suser denied: .* \(mysql-denied\) from <HOST>\s*$
+failregex = ^%(__prefix_line)suser denied: (?:\S+|.*?) \(mysql-denied\) from <HOST>\s*$
 
 ignoreregex =
 
 
 # Author: Pavel Mihadyuk
+# Regex fixes: Serg G. Brester
diff --git a/fail2ban/tests/files/logs/phpmyadmin-syslog b/fail2ban/tests/files/logs/phpmyadmin-syslog
index eef6e3b2..f32a2476 100644
--- a/fail2ban/tests/files/logs/phpmyadmin-syslog
+++ b/fail2ban/tests/files/logs/phpmyadmin-syslog
@@ -1,2 +1,2 @@
-# failJSON: { "time": "2017-08-22T14:50:22", "match": true , "host": "81.62.21.201" }
-Aug 22 14:50:22 eurostream phpMyAdmin[16358]: user denied: root (mysql-denied) from 81.62.21.201
+# failJSON: { "time": "2004-08-22T14:50:22", "match": true , "host": "192.0.2.1" }
+Aug 22 14:50:22 eurostream phpMyAdmin[16358]: user denied: root (mysql-denied) from 192.0.2.1
diff --git a/fail2ban/tests/files/logs/phpmyadmin-syslog.conf b/fail2ban/tests/files/logs/phpmyadmin-syslog.conf
deleted file mode 100644
index eef6e3b2..00000000
--- a/fail2ban/tests/files/logs/phpmyadmin-syslog.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# failJSON: { "time": "2017-08-22T14:50:22", "match": true , "host": "81.62.21.201" }
-Aug 22 14:50:22 eurostream phpMyAdmin[16358]: user denied: root (mysql-denied) from 81.62.21.201

From 9b8563f35e88a764e60b61cfdf162dfaf726b0d8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 1 Sep 2017 09:56:21 +0200
Subject: [PATCH 493/623] - fixes regex for message `imap-login: Disconnected
 (auth failed, X attempts) ...` has to many variations on additional info
 after `<HOST>`,   leave it end-anchored because variable part `user=<[^>]*>`
 (before `<HOST>`) to avoid injecting, but can be safe rewritten using `[^>]*`
 in opposite to "greedy" `user=<[^>]*>`. - introduces mode `aggressive` and
 extends regex for this mode to match:   * no auth attempts (previously
 removed in gh-601, because of lots of false positives on misconfigured MTAs) 
  * disconnected before auth was ready   * client didn't finish SASL auth

---
 config/filter.d/dovecot.conf      | 17 ++++++++++++++---
 fail2ban/tests/files/logs/dovecot | 16 ++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index 42ab2c88..fc649d3a 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -13,9 +13,22 @@ _daemon = (dovecot(-auth)?|auth)
 prefregex = ^%(__prefix_line)s(%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
+            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
             ^pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
             ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
+            <mdre-<mode>>
+
+mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+
+mdre-normal = 
+
+# Parameter `mode` - `normal` or `aggressive`.
+# Aggressive mode can be used to match log-entries like:
+#   'no auth attempts', 'disconnected before auth was ready', 'client didn't finish SASL auth'.
+# Note it may produce lots of false positives on misconfigured MTAs.
+# Ex.:
+# filter = dovecot[mode=aggressive]
+mode = normal
 
 ignoreregex = 
 
@@ -27,8 +40,6 @@ datepattern = {^LN-BEG}TAI64N
 # DEV Notes:
 # * the first regex is essentially a copy of pam-generic.conf
 # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016)
-# * Removed the 'no auth attempts' log lines from the matches because produces
-#    lots of false positives on misconfigured MTAs making regexp unusable
 #
 # Author: Martin Waschbuesch
 #         Daniel Black (rewrote with begin and end anchors)
diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot
index 1614ff8c..34c91d42 100644
--- a/fail2ban/tests/files/logs/dovecot
+++ b/fail2ban/tests/files/logs/dovecot
@@ -81,3 +81,19 @@ Mar 23 06:10:52 auth: Info: ldap(dog,52.37.139.121,): invalid credentials
 Jul 26 11:11:21 hostname dovecot: imap-login: Disconnected: Too many invalid commands (tried to use disallowed plaintext auth): user=<test>, rip=192.0.2.1, lip=192.168.1.1, session=<S5dIdTFCDKUWWMbU>
 # failJSON: { "time": "2005-07-26T11:12:19", "match": true , "host": "192.0.2.2" }
 Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid commands (auth failed, 1 attempts in 17 secs): user=<test>, method=PLAIN, rip=192.0.2.2, lip=192.168.1.1, TLS, session=<g3ZKeDECFqlWWMbU>
+
+# failJSON: { "time": "2004-08-28T06:38:51", "match": true , "host": "192.0.2.3" }
+Aug 28 06:38:51 s166-62-100-187 dovecot: imap-login: Disconnected (auth failed, 1 attempts in 9 secs): user=<administrator@example.com>, method=PLAIN, rip=192.0.2.3, lip=192.168.1.2, TLS: Disconnected, TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)
+
+# ---------------------------------------
+# Test-cases of aggressive mode:
+# ---------------------------------------
+
+# filterOptions: [{"mode": "aggressive"}]
+
+# failJSON: { "time": "2004-08-29T16:06:58", "match": true , "host": "192.0.2.5" }
+Aug 29 16:06:58 s166-62-100-187 dovecot: imap-login: Disconnected (disconnected before auth was ready, waited 0 secs): user=<>, rip=192.0.2.5, lip=192.168.1.2, TLS handshaking: SSL_accept() syscall failed: Connection reset by peer
+# failJSON: { "time": "2004-08-31T16:15:10", "match": true , "host": "192.0.2.6" }
+Aug 31 16:15:10 s166-62-100-187 dovecot: imap-login: Disconnected (client didn't finish SASL auth, waited 2 secs): user=<>, method=PLAIN, rip=192.0.2.6, lip=192.168.1.2, TLS: SSL_read() syscall failed: Connection reset by peer, TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)
+# failJSON: { "time": "2004-08-31T16:21:53", "match": true , "host": "192.0.2.7" }
+Aug 31 16:21:53 s166-62-100-187 dovecot: imap-login: Disconnected (no auth attempts in 4 secs): user=<>, rip=192.0.2.7, lip=192.168.1.2, TLS handshaking: SSL_accept() syscall failed: Connection reset by peer

From 2cfc53c08efcbc9b1d1f5c5da6d719b3867618ce Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 1 Sep 2017 10:25:09 +0200
Subject: [PATCH 494/623] remove capturing groups

---
 config/filter.d/dovecot.conf | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index fc649d3a..4df7c752 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -8,17 +8,17 @@ before = common.conf
 [Definition]
 
 _auth_worker = (?:dovecot: )?auth(?:-worker)?
-_daemon = (dovecot(-auth)?|auth)
+_daemon = (?:dovecot(?:-auth)?|auth)
 
-prefregex = ^%(__prefix_line)s(%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
-            ^pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
+            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
             ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
             <mdre-<mode>>
 
-mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
 
 mdre-normal = 
 

From c312962029bdae1d3908c04fe86b7717c6938a34 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 1 Sep 2017 10:57:41 +0200
Subject: [PATCH 495/623] filter.d/dovecot.conf: partially cherry-pick to 0.9
 PR #1880 from sebres/0.10-fix-dovecot-regex
 (d926e11a5c0d5a0d3a724f04f7b4540c6d873b4b) fixed failregex (without new mode
 aggressive)

---
 config/filter.d/dovecot.conf      | 13 ++++++++-----
 fail2ban/tests/files/logs/dovecot |  3 +++
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index d4b49576..cf217bcb 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -7,12 +7,15 @@ before = common.conf
 
 [Definition]
 
-_daemon = (auth|dovecot(-auth)?|auth-worker)
+_auth_worker = (?:dovecot: )?auth(?:-worker)?
+_daemon = (?:dovecot(?:-auth)?|auth)
 
-failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
-            ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
-            ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:Info: )?[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
+__prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?
+
+failregex = ^%(__prefregex)sauthentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
+            ^%(__prefregex)s(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^%(__prefregex)spam\(\S+,<HOST>\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
+            ^%(__prefregex)s[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot
index 1614ff8c..f564b76e 100644
--- a/fail2ban/tests/files/logs/dovecot
+++ b/fail2ban/tests/files/logs/dovecot
@@ -81,3 +81,6 @@ Mar 23 06:10:52 auth: Info: ldap(dog,52.37.139.121,): invalid credentials
 Jul 26 11:11:21 hostname dovecot: imap-login: Disconnected: Too many invalid commands (tried to use disallowed plaintext auth): user=<test>, rip=192.0.2.1, lip=192.168.1.1, session=<S5dIdTFCDKUWWMbU>
 # failJSON: { "time": "2005-07-26T11:12:19", "match": true , "host": "192.0.2.2" }
 Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid commands (auth failed, 1 attempts in 17 secs): user=<test>, method=PLAIN, rip=192.0.2.2, lip=192.168.1.1, TLS, session=<g3ZKeDECFqlWWMbU>
+
+# failJSON: { "time": "2004-08-28T06:38:51", "match": true , "host": "192.0.2.3" }
+Aug 28 06:38:51 s166-62-100-187 dovecot: imap-login: Disconnected (auth failed, 1 attempts in 9 secs): user=<administrator@example.com>, method=PLAIN, rip=192.0.2.3, lip=192.168.1.2, TLS: Disconnected, TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)

From a90f6c4ae817873c2e3090a0dd24c457636b2e6e Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Tue, 29 Mar 2016 21:25:25 +1100
Subject: [PATCH 496/623] added zoneminder jail and filter

# Conflicts:
#	config/jail.conf
---
 config/jail.conf | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/config/jail.conf b/config/jail.conf
index cf652fe2..870d7f43 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -866,3 +866,14 @@ port    = http,https
 filter  = phpmyadmin-syslog
 logpath = %(syslog_authpriv)s
 backend = %(syslog_backend)s
+
+
+[zoneminder]
+# Zoneminder HTTP/HTTPS web interface auth
+# Logs auth failures to apache2 error log
+enabled = false
+port    = http,https
+filter  = zoneminder
+logpath = /var/log/apache*/*error.log
+maxretry = 3
+

From 08878d22dda512b2d01d0649a1cf0e86023d9b2d Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Tue, 29 Mar 2016 21:31:26 +1100
Subject: [PATCH 497/623] added zoneminder.conf filter

---
 config/filter.d/zoneminder.conf | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 config/filter.d/zoneminder.conf

diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf
new file mode 100644
index 00000000..00da76d9
--- /dev/null
+++ b/config/filter.d/zoneminder.conf
@@ -0,0 +1,23 @@
+# Fail2Ban filter for Zoneminder login failures
+#
+
+[Definition]
+
+# patern :      [client 10.1.1.1:38022] WAR [Login denied for user "test"], referer: https://zoneminderurl/
+#
+#
+# Option:  failregex
+# Notes.:  regex to match the password failure messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+
+
+failregex = [[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user
+
+ignoreregex =
+
+# Notes:
+#	Tested on Zoneminder 1.29.0
+#
+# Author: John Marzella

From 44c4496e49a0a37995ba452e52f42f957c2422dc Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Tue, 29 Mar 2016 21:43:36 +1100
Subject: [PATCH 498/623] added sample log files

---
 fail2ban/tests/files/logs/zoneminder | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 fail2ban/tests/files/logs/zoneminder

diff --git a/fail2ban/tests/files/logs/zoneminder b/fail2ban/tests/files/logs/zoneminder
new file mode 100644
index 00000000..c27a0c7b
--- /dev/null
+++ b/fail2ban/tests/files/logs/zoneminder
@@ -0,0 +1,6 @@
+[Mon Mar 28 10:51:24.201977 2016] [:error] [pid 29646] [client 10.1.1.2:46454] INF [Login successful for user "gooduser1"]
+[Mon Mar 28 16:50:45.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
+[Mon Mar 28 16:50:46.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
+[Mon Mar 28 16:50:47.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
+[Mon Mar 28 16:50:48.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
+[Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/

From 4d8ba7b668de0e3cbf82c7986d60fc6dcfa70d90 Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Wed, 30 Mar 2016 15:36:48 +1100
Subject: [PATCH 499/623] fixed test log file

---
 fail2ban/tests/files/logs/zoneminder | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/files/logs/zoneminder b/fail2ban/tests/files/logs/zoneminder
index c27a0c7b..63d8bbd4 100644
--- a/fail2ban/tests/files/logs/zoneminder
+++ b/fail2ban/tests/files/logs/zoneminder
@@ -1,6 +1,2 @@
-[Mon Mar 28 10:51:24.201977 2016] [:error] [pid 29646] [client 10.1.1.2:46454] INF [Login successful for user "gooduser1"]
-[Mon Mar 28 16:50:45.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
-[Mon Mar 28 16:50:46.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
-[Mon Mar 28 16:50:47.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
-[Mon Mar 28 16:50:48.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
-[Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "baduser1"], referer: https://zoneminder/
+# failJSON { "time": "2016-03-28T16:50:49", "match": true , "host": "10.1.1.1" }
+[Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/

From 776d463e9299283318e53145eceded5256bc2703 Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Wed, 30 Mar 2016 15:46:37 +1100
Subject: [PATCH 500/623] added missing colon to failJSON

---
 fail2ban/tests/files/logs/zoneminder | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/zoneminder b/fail2ban/tests/files/logs/zoneminder
index 63d8bbd4..abd49869 100644
--- a/fail2ban/tests/files/logs/zoneminder
+++ b/fail2ban/tests/files/logs/zoneminder
@@ -1,2 +1,2 @@
-# failJSON { "time": "2016-03-28T16:50:49", "match": true , "host": "10.1.1.1" }
+# failJSON: { "time": "2016-03-28T16:50:49", "match": true , "host": "10.1.1.1" }
 [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/

From 3d45fd2713b32c423d1f499fb5509f6be9c10a0b Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Fri, 1 Apr 2016 22:16:30 +1100
Subject: [PATCH 501/623] implemented yarikoptic's suggestions in fail2ban pull
 request #1376

---
 config/filter.d/zoneminder.conf | 4 ++--
 config/jail.conf                | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf
index 00da76d9..83b1463a 100644
--- a/config/filter.d/zoneminder.conf
+++ b/config/filter.d/zoneminder.conf
@@ -3,7 +3,7 @@
 
 [Definition]
 
-# patern :      [client 10.1.1.1:38022] WAR [Login denied for user "test"], referer: https://zoneminderurl/
+# pattern: [client 10.1.1.1:38022] WAR [Login denied for user "test"], referer: https://zoneminderurl/
 #
 #
 # Option:  failregex
@@ -13,7 +13,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 
 
-failregex = [[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user
+failregex = ^[[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user \S*], referer: \S*$
 
 ignoreregex =
 
diff --git a/config/jail.conf b/config/jail.conf
index 870d7f43..83a41c51 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -873,7 +873,6 @@ backend = %(syslog_backend)s
 # Logs auth failures to apache2 error log
 enabled = false
 port    = http,https
-filter  = zoneminder
-logpath = /var/log/apache*/*error.log
+logpath = %(apache_error_log)s
 maxretry = 3
 

From 5c3a666380d8591976b757c53a17c374d17f30dd Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Fri, 1 Apr 2016 22:30:41 +1100
Subject: [PATCH 502/623] fixed incomplete regex after adding anchors

---
 config/filter.d/zoneminder.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf
index 83b1463a..c804626e 100644
--- a/config/filter.d/zoneminder.conf
+++ b/config/filter.d/zoneminder.conf
@@ -13,7 +13,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 
 
-failregex = ^[[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user \S*], referer: \S*$
+failregex = ^.*? [[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user \S*], referer: \S*$
 
 ignoreregex =
 

From 7013729a1f86de4f38da5297cfb5eae2c4b4abf3 Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Thu, 28 Apr 2016 14:19:29 +1000
Subject: [PATCH 503/623] removed redundant options for zoneminder from
 jail.conf

---
 config/jail.conf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 83a41c51..29ad4c96 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -871,8 +871,6 @@ backend = %(syslog_backend)s
 [zoneminder]
 # Zoneminder HTTP/HTTPS web interface auth
 # Logs auth failures to apache2 error log
-enabled = false
 port    = http,https
 logpath = %(apache_error_log)s
-maxretry = 3
 

From ac95449bbb9c46e879b33540885e7d77ecd01814 Mon Sep 17 00:00:00 2001
From: john <john@terabyteit.com.au>
Date: Sat, 30 Apr 2016 15:26:36 +1000
Subject: [PATCH 504/623] changed zoneminder regex as per Sebres and yarikoptic
 recommendations

---
 config/filter.d/zoneminder.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf
index c804626e..aa3b4e6d 100644
--- a/config/filter.d/zoneminder.conf
+++ b/config/filter.d/zoneminder.conf
@@ -3,7 +3,7 @@
 
 [Definition]
 
-# pattern: [client 10.1.1.1:38022] WAR [Login denied for user "test"], referer: https://zoneminderurl/
+# pattern: [Wed Apr 27 23:12:07.736196 2016] [:error] [pid 2460] [client 10.1.1.1:47296] WAR [Login denied for user "test"], referer: https://zoneminderurl/index.php
 #
 #
 # Option:  failregex
@@ -13,7 +13,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 
 
-failregex = ^.*? [[]client <HOST>:\d\d\d\d\d] WAR [[]Login denied for user \S*], referer: \S*$
+failregex = ^\[\](?: \[:error\])?(?: \[pid \d+\])? \[client <HOST>:\d+\] WAR \[Login denied for user "[^"]*"\]
 
 ignoreregex =
 

From 4163f329688e89c3d551423a0e3d9be02adb9897 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 4 Sep 2017 11:48:01 +0200
Subject: [PATCH 505/623] small review, prefix replaced with
 `%(_apache_error_client)s` from apache-common.conf include

---
 config/filter.d/zoneminder.conf | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf
index aa3b4e6d..cc82755a 100644
--- a/config/filter.d/zoneminder.conf
+++ b/config/filter.d/zoneminder.conf
@@ -1,5 +1,7 @@
 # Fail2Ban filter for Zoneminder login failures
-#
+
+[INCLUDES]
+before = apache-common.conf
 
 [Definition]
 
@@ -7,13 +9,9 @@
 #
 #
 # Option:  failregex
-# Notes.:  regex to match the password failure messages in the logfile. The
-#          host must be matched by a group named "host". The tag "<HOST>" can
-#          be used for standard IP/hostname matching and is only an alias for
-#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+# Notes.:  regex to match the password failure messages in the logfile.
 
-
-failregex = ^\[\](?: \[:error\])?(?: \[pid \d+\])? \[client <HOST>:\d+\] WAR \[Login denied for user "[^"]*"\]
+failregex = ^%(_apache_error_client)s WAR \[Login denied for user "[^"]*"\]
 
 ignoreregex =
 

From fafefc029393bd33d6d593d4adbdea957a86e3d8 Mon Sep 17 00:00:00 2001
From: Vladimir Chumak <coder.ua@gmail.com>
Date: Sun, 3 Sep 2017 14:07:55 +0300
Subject: [PATCH 506/623] Add Jorgee Vulnerability Scanner protect

Details for Jorgee Vulnerability Scanner: https://www.symantec.com/security_response/attacksignatures/detail.jsp?asid=30164
---
 config/filter.d/apache-badbots.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf
index 48b30666..c00852c4 100644
--- a/config/filter.d/apache-badbots.conf
+++ b/config/filter.d/apache-badbots.conf
@@ -7,7 +7,7 @@
 
 [Definition]
 
-badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider
+badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|Jorgee|Mozilla/5\.0 Jorgee
 badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots&#44; \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00
 
 failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$

From 4bc226a692b06eb61e9a54146cd70ca91e0bf9ea Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Sep 2017 10:59:16 +0200
Subject: [PATCH 507/623] optimized regex

---
 config/filter.d/apache-badbots.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf
index c00852c4..cc7e6549 100644
--- a/config/filter.d/apache-badbots.conf
+++ b/config/filter.d/apache-badbots.conf
@@ -7,7 +7,7 @@
 
 [Definition]
 
-badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|Jorgee|Mozilla/5\.0 Jorgee
+badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|(?:Mozilla/\d+\.\d+ )?Jorgee
 badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots&#44; \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00
 
 failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$

From 2cd02b731b3e79b9fd052cc348416124cc57f785 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 7 Sep 2017 15:28:46 +0200
Subject: [PATCH 508/623] filter.d/exim.conf: fixed failregex for case of
 `D=0s`

Closes gh-1886
---
 config/filter.d/exim.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 7848fe00..eb56feff 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -19,7 +19,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
             ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S+s(?: C=\S*)?\s*$
+            ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 
 ignoreregex = 

From db121a6f8577c1fb348d8ed7b0f6c1a999c6a28c Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 7 Sep 2017 15:32:35 +0200
Subject: [PATCH 509/623] Update exim

Test case covers flood attempts with `D=0s`
---
 fail2ban/tests/files/logs/exim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index 8f3ac1bb..f87d393b 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -53,7 +53,7 @@
 # failJSON: { "time": "2016-03-21T06:38:05", "match": true , "host": "49.212.207.15" }
 2016-03-21 06:38:05 [5718] no MAIL in SMTP connection from www3005.sakura.ne.jp [49.212.207.15]:28890 I=[172.89.0.6]:25 D=21s C=EHLO,STARTTLS
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
-2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=10s
+2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116]:2056 I=[172.89.0.6]:25 D=0s
 # failJSON: { "time": "2016-03-21T06:57:36", "match": true , "host": "122.165.71.116" }
 2016-03-21 06:57:36 [5908] no MAIL in SMTP connection from [122.165.71.116] I=[172.89.0.6]:25 D=10s
 # failJSON: { "time": "2016-03-21T04:07:49", "match": true , "host": "174.137.147.204" }

From fd83260bd8973e704f6056dca86e9360b3a876bb Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 7 Sep 2017 18:51:08 +0200
Subject: [PATCH 510/623] jail "pass2allow-ftp" should supply blocktype to
 action

closes gh-1884
---
 config/jail.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/jail.conf b/config/jail.conf
index e3e89ff0..7c696176 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -840,6 +840,7 @@ filter       = apache-pass[knocking_url="%(knocking_url)s"]
 logpath      = %(apache_access_log)s
 blocktype    = RETURN
 returntype   = DROP
+action       = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s]
 bantime      = 1h
 maxretry     = 1
 findtime     = 1

From b698a749029af2c216268aa29a1228f21e87bfb7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 7 Sep 2017 19:07:06 +0200
Subject: [PATCH 511/623] introduces new command-line options `--dp`,
 `--dump-pretty` to dump the configuration using more human readable
 representation; allow dump of configuration, also if log-file is not
 available (warning only)

---
 fail2ban/client/configurator.py          |  4 ++--
 fail2ban/client/fail2bancmdline.py       | 26 +++++++++++++++++-------
 fail2ban/client/jailreader.py            |  9 +++++---
 fail2ban/tests/fail2banclienttestcase.py |  6 +++++-
 4 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py
index e8472ac1..fe735251 100644
--- a/fail2ban/client/configurator.py
+++ b/fail2ban/client/configurator.py
@@ -76,9 +76,9 @@ class Configurator:
 		self.__fail2ban.getOptions(updateMainOpt)
 		return self.__jails.getOptions(jail, ignoreWrong=ignoreWrong)
 		
-	def convertToProtocol(self):
+	def convertToProtocol(self, allow_no_files=False):
 		self.__streams["general"] = self.__fail2ban.convert()
-		self.__streams["jails"] = self.__jails.convert()
+		self.__streams["jails"] = self.__jails.convert(allow_no_files=allow_no_files)
 	
 	def getConfigStream(self):
 		cmds = list()
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index f213d037..401aa9b6 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -102,6 +102,7 @@ class Fail2banCmdLine():
 		output("    --logtarget <FILE>|STDOUT|STDERR|SYSLOG")
 		output("    --syslogsocket auto|<FILE>")
 		output("    -d                      dump configuration. For debugging")
+		output("    --dp, --dump-pretty     dump the configuration using more human readable representation")
 		output("    -t, --test              test configuration (can be also specified with start parameters)")
 		output("    -i                      interactive mode")
 		output("    -v                      increase verbosity")
@@ -137,8 +138,8 @@ class Fail2banCmdLine():
 				self._conf["pidfile"] = opt[1]
 			elif o.startswith("--log") or o.startswith("--sys"):
 				self._conf[ o[2:] ] = opt[1]
-			elif o == "-d":
-				self._conf["dump"] = True
+			elif o in ["-d", "--dp", "--dump-pretty"]:
+				self._conf["dump"] = True if o == "-d" else 2
 			elif o == "-t" or o == "--test":
 				self.cleanConfOnly = True
 				self._conf["test"] = True
@@ -184,7 +185,8 @@ class Fail2banCmdLine():
 			# Reads the command line options.
 			try:
 				cmdOpts = 'hc:s:p:xfbdtviqV'
-				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'timeout=', 'str2sec=', 'help', 'version']
+				cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async',
+					'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty']
 				optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts)
 			except getopt.GetoptError:
 				self.dispUsage()
@@ -240,7 +242,10 @@ class Fail2banCmdLine():
 				if readcfg:
 					ret, stream = self.readConfig()
 					readcfg = False
-				self.dumpConfig(stream)
+				if stream is not None:
+					self.dumpConfig(stream, self._conf["dump"] == 2)
+				else: # pragma: no cover
+					output("ERROR: The configuration stream failed because of the invalid syntax.")
 				if not self._conf.get("test", False):
 					return ret
 
@@ -275,7 +280,8 @@ class Fail2banCmdLine():
 			self.configurator.readAll()
 			ret = self.configurator.getOptions(jail, self._conf, 
 				ignoreWrong=not self.cleanConfOnly)
-			self.configurator.convertToProtocol()
+			self.configurator.convertToProtocol(
+				allow_no_files=self._conf.get("dump", False))
 			stream = self.configurator.getConfigStream()
 		except Exception as e:
 			logSys.error("Failed during configuration: %s" % e)
@@ -283,9 +289,15 @@ class Fail2banCmdLine():
 		return ret, stream
 
 	@staticmethod
-	def dumpConfig(cmd):
+	def dumpConfig(cmd, pretty=False):
+		if pretty:
+			from pprint import pformat
+			def _output(s):
+				output(pformat(s, width=1000, indent=2))
+		else:
+			_output = output
 		for c in cmd:
-			output(c)
+			_output(c)
 		return True
 
 	#
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index ce0ed3b6..ffc28752 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -235,9 +235,12 @@ class JailReader(ConfigReader):
 						found_files += 1
 						stream.append(
 							["set", self.__name, "addlogpath", p, tail])
-				if not (found_files or allow_no_files):
-					raise ValueError(
-						"Have not found any log file for %s jail" % self.__name)
+				if not found_files:
+					msg = "Have not found any log file for %s jail" % self.__name
+					if not allow_no_files:
+						raise ValueError(msg)
+					logSys.warning(msg)
+					
 			elif opt == "logencoding":
 				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index caacf63c..083ce815 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -426,7 +426,11 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, True)
 		self.execSuccess(startparams, "-vvd")
 		self.assertLogged("Loading files")
-		self.assertLogged("logtarget")
+		self.assertLogged("['set', 'logtarget',")
+		self.pruneLog()
+		# pretty dump:
+		self.execSuccess(startparams, "--dp")
+		self.assertLogged("['set', 'logtarget',")
 		
 	@with_tmpdir
 	@with_kill_srv

From e20f6204d325273eadd724e8a15401379b634d5c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 7 Sep 2017 19:09:18 +0200
Subject: [PATCH 512/623] don't put parameters starting with `known/` to the
 ready stream (intermediate options only), makes streams and dumps of
 configuration shorter and better readable

---
 fail2ban/client/actionreader.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index 559705bc..d7bba61f 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -88,11 +88,11 @@ class ActionReader(DefinitionInitConfigReader):
 		stream.append(head + ["addaction", self._name])
 		multi = []
 		for opt, optval in opts.iteritems():
-			if opt in self._configOpts:
+			if opt in self._configOpts and not opt.startswith('known/'):
 				multi.append([opt, optval])
 		if self._initOpts:
 			for opt, optval in self._initOpts.iteritems():
-				if opt not in self._configOpts:
+				if opt not in self._configOpts and not opt.startswith('known/'):
 					multi.append([opt, optval])
 		if len(multi) > 1:
 			stream.append(["multi-set", self._jailName, "action", self._name, multi])

From 462b534469dcdd852e6c551ad71ea0549e63ecc2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 7 Sep 2017 19:21:45 +0200
Subject: [PATCH 513/623] restrict saving of previous known values to
 section-related (don't overwrite with the values of other sections,
 especially like "INCLUDES", etc.)

---
 fail2ban/client/configparserinc.py | 9 ++++-----
 fail2ban/client/configreader.py    | 6 +++---
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
index b626be9b..d220ed44 100644
--- a/fail2ban/client/configparserinc.py
+++ b/fail2ban/client/configparserinc.py
@@ -20,8 +20,8 @@
 # Author: Yaroslav Halchenko
 # Modified: Cyril Jaquier
 
-__author__ = 'Yaroslav Halhenko'
-__copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko'
+__author__ = 'Yaroslav Halhenko, Serg G. Brester (aka sebres)'
+__copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko, 2015 Serg G. Brester (aka sebres)'
 __license__ = 'GPL'
 
 import os
@@ -150,7 +150,7 @@ after = 1.conf
 					if seclwr == 'known':
 						# try get raw value from known options:
 						try:
-							v = self._sections['KNOWN'][opt]
+							v = self._sections['KNOWN/'+section][opt]
 						except KeyError:
 							# fallback to default:
 							try:
@@ -297,7 +297,6 @@ after = 1.conf
 					# merge defaults and all sections to self:
 					alld.update(cfg.get_defaults())
 					for n, s in cfg.get_sections().iteritems():
-						curalls = alls
 						# conditional sections
 						cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
 						if cond:
@@ -313,7 +312,7 @@ after = 1.conf
 						s2 = alls.get(n)
 						if isinstance(s2, dict):
 							# save previous known values, for possible using in local interpolations later:
-							self.merge_section('KNOWN', s2, '')
+							self.merge_section('KNOWN/'+n, s2, '')
 							# merge section
 							s2.update(s)
 						else:
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index 381af759..577a5a16 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -20,8 +20,8 @@
 # Author: Cyril Jaquier
 # Modified by: Yaroslav Halchenko (SafeConfigParserWithIncludes)
 
-__author__ = "Cyril Jaquier"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
+__author__ = "Cyril Jaquier, Yaroslav Halchenko, Serg G. Brester (aka sebres)"
+__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2007 Yaroslav Halchenko, 2015 Serg G. Brester (aka sebres)"
 __license__ = "GPL"
 
 import glob
@@ -110,7 +110,7 @@ class ConfigReader():
 
 	def sections(self):
 		try:
-			return (n for n in self._cfg.sections() if n != 'KNOWN')
+			return (n for n in self._cfg.sections() if not n.startswith('KNOWN/'))
 		except AttributeError:
 			return []
 

From 983b128c5439cacfbc560c600266051b4bf6de6e Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 8 Sep 2017 11:07:48 +0200
Subject: [PATCH 514/623] Update ChangeLog

several fixes of 0.9th branch
---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 32b8999d..ae7d51c3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -25,7 +25,10 @@ releases.
 * filter.d/apache-overflows.conf:
     - Fixes resources greedy expression (see gh-1790);
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
+* filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
 * filter.d/asterisk.conf - fixed failregex AMI Asterisk authentification failed (see gh-1302)
+* filter.d/dovecot.conf - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
+* filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
 
 ### New Features
 
@@ -33,6 +36,7 @@ releases.
 * action.d/cloudflare.conf - Cloudflare API v4 implementation (gh-1651)
 * filter.d/kerio.conf - filter extended with new rules (see gh-1455)
 * filter.d/phpmyadmin-syslog.conf - new filter for phpMyAdmin using syslog for auth logging
+* filter.d/zoneminder.conf - new filter for ZoneMinder (gh-1376)
 
 
 ver. 0.9.7 (2017/05/11) - awaiting-victory

From 8be4569d51350728446e7a92c8027f73e116afc8 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 8 Sep 2017 11:32:08 +0200
Subject: [PATCH 515/623] Update ChangeLog

several fixes of 0.10th branch
---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5bdb9be4..c56ca6ec 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,10 +35,25 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 
 ### Fixes
+* jail "pass2allow-ftp" supply blocktype and returntype parameters to the action (gh-1884)
+* avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
+  'LANGUAGE', 'LC_ALL', 'LC_CTYPE', and 'LANG', see gh-1587).
+* action.d/pf.conf: several fixes for pf-action like anchoring, etc. (see gh-1866, gh-1867);
+* fixed ignorself issue "Retrieving own IPs of localhost failed: inet_pton() argument 2 must be string, not int" (see gh-1865);
+* fixed tags `<fq-hostname>` and `<sh-hostname>`, could be used without ticket (a. g. in `actionstart` etc., gh-1859).
+
+* setup.py: fixed several setup facilities (gh-1874):
+  - don't check return code by dry-run: returns 256 on some python/setuptool versions;
+  - `files/fail2ban.service` renamed as template to `files/fail2ban.service.in`;
+  - setup process generates `build/fail2ban.service` from `files/fail2ban.service.in` using distribution related bin-path;
+  - bug-fixing by running setup with option `--dry-run`;
 
 ### New Features
+* introduced new command-line options `--dp`, `--dump-pretty` to dump the configuration using more
+  human readable representation (opposite to `-d`);
 
 ### Enhancements
+* filter.d/dovecot.conf: introduced mode `aggressive` for cases like "disconnected before auth was ready" (gh-1880)
 
 
 ver. 0.10.0 (2017/08/09) - long-awaited 0.10th version

From 2ce0ffb97752f51199fb85f56167ecf7079a7a02 Mon Sep 17 00:00:00 2001
From: Louis Sautier <sautier.louis@gmail.com>
Date: Sun, 10 Sep 2017 21:37:03 +0200
Subject: [PATCH 516/623] Fix Gentoo init script's shebang

Use openrc-run instead of runscript.
https://github.com/OpenRC/openrc/commit/5d5856c193768d24f11d5f0533e48c39526aef5c
---
 ChangeLog          | 1 +
 files/gentoo-initd | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c56ca6ec..5beae7ec 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,6 +35,7 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 
 ### Fixes
+* fix Gentoo init script's shebang to use openrc-run instead of runscript
 * jail "pass2allow-ftp" supply blocktype and returntype parameters to the action (gh-1884)
 * avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
   'LANGUAGE', 'LC_ALL', 'LC_CTYPE', and 'LANG', see gh-1587).
diff --git a/files/gentoo-initd b/files/gentoo-initd
index c4d8675c..d4e52bcb 100755
--- a/files/gentoo-initd
+++ b/files/gentoo-initd
@@ -1,4 +1,4 @@
-#!/sbin/runscript
+#!/sbin/openrc-run
 # This file is part of Fail2Ban.
 #
 # Fail2Ban is free software; you can redistribute it and/or modify

From 72ad904f584be4df44963c2b058a29d9f076d013 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Mon, 11 Sep 2017 12:22:43 +0200
Subject: [PATCH 517/623] Update ChangeLog

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 5beae7ec..27be6498 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,7 +35,7 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 
 ### Fixes
-* fix Gentoo init script's shebang to use openrc-run instead of runscript
+* fix Gentoo init script's shebang to use openrc-run instead of runscript (gh-1891)
 * jail "pass2allow-ftp" supply blocktype and returntype parameters to the action (gh-1884)
 * avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
   'LANGUAGE', 'LC_ALL', 'LC_CTYPE', and 'LANG', see gh-1587).

From 152c9d27d598414ad5a3a5d0932e0310c5ecdc57 Mon Sep 17 00:00:00 2001
From: Louis Sautier <sautier.louis@gmail.com>
Date: Mon, 11 Sep 2017 23:22:01 +0200
Subject: [PATCH 518/623] Fix nftables actions for IPv6 addresses, fixes #1893

* add [Init?family=inet6] to nftables-common.conf and make nftable
  expressions more modular
* change "ip protocol" to "meta l4proto" in nftables-allports.conf
  since the former only works for IPv4
---
 ChangeLog                              |  1 +
 config/action.d/nftables-allports.conf |  2 +-
 config/action.d/nftables-common.conf   | 30 ++++++++++++++++++++------
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 27be6498..e3c10de2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,6 +35,7 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 
 ### Fixes
+* fix nftables actions for IPv6 addresses (gh-1893)
 * fix Gentoo init script's shebang to use openrc-run instead of runscript (gh-1891)
 * jail "pass2allow-ftp" supply blocktype and returntype parameters to the action (gh-1884)
 * avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf
index afd0ca84..6c69da39 100644
--- a/config/action.d/nftables-allports.conf
+++ b/config/action.d/nftables-allports.conf
@@ -17,6 +17,6 @@ before = nftables-common.conf
 # Notes.:  additional expressions for nftables filter rule
 # Values:  nftables expressions
 #
-nftables_mode = ip protocol <protocol>
+nftables_mode = meta l4proto <protocol>
 
 [Init]
diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf
index 80657c5c..83311087 100644
--- a/config/action.d/nftables-common.conf
+++ b/config/action.d/nftables-common.conf
@@ -28,11 +28,11 @@ nftables_mode = <protocol> dport \{ <port> \}
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = <nftables> add set <nftables_family> <nftables_table> f2b-<name> \{ type <nftables_type>\; \}
-              <nftables> insert rule <nftables_family> <nftables_table> <chain> %(nftables_mode)s ip saddr @f2b-<name> <blocktype>
+actionstart = <nftables> add set <nftables_family> <nftables_table> <set_name> \{ type <nftables_type>\; \}
+              <nftables> insert rule <nftables_family> <nftables_table> <chain> %(nftables_mode)s <address_family> saddr @<set_name> <blocktype>
 
 _nft_list = <nftables> --handle --numeric list chain <nftables_family> <nftables_table> <chain>
-_nft_get_handle_id = grep -m1 'ip saddr @f2b-<name> <blocktype> # handle' | grep -oe ' handle [0-9]*'
+_nft_get_handle_id = grep -m1 '<address_family> saddr @<set_name> <blocktype> # handle' | grep -oe ' handle [0-9]*'
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -40,13 +40,13 @@ _nft_get_handle_id = grep -m1 'ip saddr @f2b-<name> <blocktype> # handle' | grep
 #
 actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s)
              <nftables> delete rule <nftables_family> <nftables_table> <chain> $HANDLE_ID
-             <nftables> delete set <nftables_family> <nftables_table> f2b-<name>
+             <nftables> delete set <nftables_family> <nftables_table> <set_name>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = <nftables> list chain <nftables_family> <nftables_table> <chain> | grep -q '@f2b-<name>[ \t]'
+actioncheck = <nftables> list chain <nftables_family> <nftables_table> <chain> | grep -q '@<set_name>[ \t]'
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -54,7 +54,7 @@ actioncheck = <nftables> list chain <nftables_family> <nftables_table> <chain> |
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = <nftables> add element <nftables_family> <nftables_table> f2b-<name> \{ <ip> \}
+actionban = <nftables> add element <nftables_family> <nftables_table> <set_name> \{ <ip> \}
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -62,7 +62,7 @@ actionban = <nftables> add element <nftables_family> <nftables_table> f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = <nftables> delete element <nftables_family> <nftables_table> f2b-<name> \{ <ip> \}
+actionunban = <nftables> delete element <nftables_family> <nftables_table> <set_name> \{ <ip> \}
 
 [Init]
 
@@ -117,3 +117,19 @@ blocktype = reject
 # Notes.:  Actual command to be executed, including common to all calls options
 # Values:  STRING
 nftables = nft
+
+# Option: set_name
+# Notes.: The name of the nft set used to store banned addresses
+# Values: STRING
+set_name = f2b-<name>
+
+# Option: address_family
+# Notes.: The family of the banned addresses
+# Values: [ ip | ip6 ]
+address_family = ip
+
+[Init?family=inet6]
+
+nftables_type = ipv6_addr
+set_name = f2b-<name>6
+address_family = ip6

From 6149df521643151d87deb738a0b893821deaa411 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 12 Sep 2017 09:27:16 +0200
Subject: [PATCH 519/623] Update ChangeLog

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index e3c10de2..4e1d0b55 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -35,7 +35,6 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
 -----------
 
 ### Fixes
-* fix nftables actions for IPv6 addresses (gh-1893)
 * fix Gentoo init script's shebang to use openrc-run instead of runscript (gh-1891)
 * jail "pass2allow-ftp" supply blocktype and returntype parameters to the action (gh-1884)
 * avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
@@ -55,6 +54,7 @@ ver. 0.10.1-dev-1 (2016/??/??) - development edition
   human readable representation (opposite to `-d`);
 
 ### Enhancements
+* nftables actions are IPv6-capable now (gh-1893)
 * filter.d/dovecot.conf: introduced mode `aggressive` for cases like "disconnected before auth was ready" (gh-1880)
 
 

From a2120a9de5ba6211e42aae24252e860a62d9d4fa Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 Oct 2017 15:31:55 +0200
Subject: [PATCH 520/623] filter.d/postfix-*.conf - added optional port regex
 (closes gh-1902)

---
 ChangeLog                              |  1 +
 config/filter.d/postfix-rbl.conf       |  3 ++-
 config/filter.d/postfix-sasl.conf      |  3 ++-
 config/filter.d/postfix.conf           | 15 ++++++++-------
 fail2ban/tests/files/logs/postfix-sasl |  2 ++
 5 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ae7d51c3..1c4620d3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -29,6 +29,7 @@ releases.
 * filter.d/asterisk.conf - fixed failregex AMI Asterisk authentification failed (see gh-1302)
 * filter.d/dovecot.conf - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
 * filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
+* filter.d/postfix-*.conf - added optional port regex (gh-1902)
 
 ### New Features
 
diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf
index 0a9078f0..10bd98d7 100644
--- a/config/filter.d/postfix-rbl.conf
+++ b/config/filter.d/postfix-rbl.conf
@@ -11,8 +11,9 @@ before = common.conf
 [Definition]
 
 _daemon = postfix(-\w+)?/smtpd
+_port = (?::\d+)?
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
 
 ignoreregex =
 
diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf
index 1a24ca94..d354b74a 100644
--- a/config/filter.d/postfix-sasl.conf
+++ b/config/filter.d/postfix-sasl.conf
@@ -8,8 +8,9 @@ before = common.conf
 [Definition]
 
 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+_port = (?::\d+)?
 
-failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
 
 ignoreregex = authentication failed: Connection lost to authentication server$
 
diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf
index 3051409b..7255384e 100644
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@@ -11,14 +11,15 @@ before = common.conf
 [Definition]
 
 _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+_port = (?::\d+)?
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 554 5\.7\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
-            ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[<HOST>\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
-            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]: 550 5\.1\.1 .*$
-            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
-            ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[<HOST>\]:?$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 554 5\.7\.1 .*$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+            ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[<HOST>\]%(_port)s: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname;
+            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]%(_port)s: 550 5\.1\.1 .*$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]%(_port)s: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+            ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[<HOST>\]%(_port)s:?$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl
index cdcb5121..fef2076b 100644
--- a/fail2ban/tests/files/logs/postfix-sasl
+++ b/fail2ban/tests/files/logs/postfix-sasl
@@ -30,3 +30,5 @@ Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: warning: astra4139.startdedicated.d
 # failJSON: { "time": "2005-08-03T15:30:49", "match": true , "host": "98.191.84.74" }
 Aug 3 15:30:49 ksusha postfix/smtpd[17041]: warning: mail.foldsandwalker.com[98.191.84.74]: SASL Plain authentication failed:
 
+# failJSON: { "time": "2005-08-04T16:47:52", "match": true , "host": "192.0.2.237", "desc": "cover optional port after host" }
+Aug 4 16:47:52 mail3 postfix/smtpd[31152]: warning: unknown[192.0.2.237]:55729: SASL LOGIN authentication failed: authentication failure

From d1fad22ac129260364c6dfc103497195a6c8984e Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 2 Oct 2017 15:59:14 +0200
Subject: [PATCH 521/623] Avoid exact duplicates by addIgnoreIP (closes
 gh-1900)

---
 fail2ban/server/filter.py        | 5 ++++-
 fail2ban/tests/servertestcase.py | 9 ++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c4f29878..c309bbb2 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -458,7 +458,10 @@ class Filter(JailThread):
 			return
 		# Create IP address object
 		ip = IPAddr(ipstr)
-
+		# Avoid exact duplicates
+		if ip in self.__ignoreIpList:
+			logSys.warn("  Ignore duplicate %r (%r), already in ignore list", ip, ipstr)
+			return
 		# log and append to ignore list
 		logSys.debug("  Add %r to ignore list (%r)", ip, ipstr)
 		self.__ignoreIpList.append(ip)
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 0b888407..1d963d0c 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -441,17 +441,16 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "addignoreip", value]),
 			(0, [value]))
-		# Will allow duplicate
-		#NOTE: Should duplicates be allowed, or silent ignore like logpath?
+		# Duplicates ignored
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "addignoreip", value]),
-			(0, [value, value]))
+			(0, [value]))
 		self.assertEqual(
 			self.transm.proceed(["get", self.jailName, "ignoreip"]),
-			(0, [value, value]))
+			(0, [value]))
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "delignoreip", value]),
-			(0, [value]))
+			(0, []))
 
 		self.assertEqual(
 			self.transm.proceed(["get", self.jailName, "ignoreself"]),

From ea36e1b3fc22f67b2faedb8cb57ae1b19d9f6e08 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 Oct 2017 09:49:35 +0200
Subject: [PATCH 522/623] filter.d/dovecot.conf: fixed failregex to recognize
 pam_authenticate failures with "Permission denied" (gh-1897)

---
 ChangeLog                         | 4 +++-
 config/filter.d/dovecot.conf      | 2 +-
 fail2ban/tests/files/logs/dovecot | 3 +++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 1c4620d3..fce432ed 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -27,7 +27,9 @@ releases.
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
 * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
 * filter.d/asterisk.conf - fixed failregex AMI Asterisk authentification failed (see gh-1302)
-* filter.d/dovecot.conf - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
+* filter.d/dovecot.conf:
+    - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
+    - extended to match pam_authenticate failures with "Permission denied" (gh-1897)
 * filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
 * filter.d/postfix-*.conf - added optional port regex (gh-1902)
 
diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index cf217bcb..9497c601 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -14,7 +14,7 @@ __prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam
 
 failregex = ^%(__prefregex)sauthentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
             ^%(__prefregex)s(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
-            ^%(__prefregex)spam\(\S+,<HOST>\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
+            ^%(__prefregex)spam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
             ^%(__prefregex)s[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
 
 ignoreregex = 
diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot
index f564b76e..f15a0de4 100644
--- a/fail2ban/tests/files/logs/dovecot
+++ b/fail2ban/tests/files/logs/dovecot
@@ -43,6 +43,9 @@ Jan 29 05:13:50 mail dovecot: auth: passwd-file(username,1.2.3.4): unknown user
 # failJSON: { "time": "2005-01-29T13:54:06", "match": true , "host": "192.0.2.5" }
 Jan 29 13:54:06 auth-worker(22401): Info: sql(admin@example.de,192.0.2.5,<n4JLdHNVngZGpV2j>): unknown user
 
+# failJSON: { "time": "2005-01-29T14:38:51", "match": true , "host": "192.0.2.6", "desc": "PAM Permission denied (gh-1897)" }
+Jan 29 14:38:51 example.com dovecot[24941]: auth-worker(30165): pam(user@example.com,192.0.2.6,<PNHQq8pZhqIKAQGd>): pam_authenticate() failed: Permission denied
+
 # failJSON: { "time": "2005-04-19T05:22:20", "match": true , "host": "80.255.3.104" }
 Apr 19 05:22:20 vm5 auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=informix rhost=80.255.3.104
 

From a4459765ef438db83a2898ba832ff7acba033e29 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 Oct 2017 14:16:14 +0200
Subject: [PATCH 523/623] pyinotify/polling: test filter reaction by delete of
 watching file, better detection of pending file (avoid errors in fail2ban.log
 during log-rotation). Closes gh-1865 for filterpyinotify ("cannot remove
 WD=2").

---
 fail2ban/server/filterpoll.py      | 11 +++++++++--
 fail2ban/server/filterpyinotify.py | 14 ++++++++++++--
 fail2ban/tests/filtertestcase.py   | 12 ++++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index e785a0e4..5b736a4e 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -164,9 +164,13 @@ class FilterPoll(FileFilter):
 				return False
 			# log error:
 			if self.__file404Cnt[filename] < 2:
-				logSys.error("Unable to get stat on %s because of: %s",
+				if e.errno == 2:
+					logSys.debug("Log absence detected (possibly rotation) for %s, reason: %s",
+							 filename, e)
+				else: # pragma: no cover
+					logSys.error("Unable to get stat on %s because of: %s",
 							 filename, e, 
-							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+							 exc_info=e.errno != 2 and logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
 			self.commonError()
@@ -175,3 +179,6 @@ class FilterPoll(FileFilter):
 				self.__file404Cnt[filename] = 0
 				self.delLogPath(filename)
 			return False
+
+	def getPendingPaths(self):
+		return self.__file404Cnt.keys()
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 10f51847..e59c203e 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -161,6 +161,9 @@ class FilterPyinotify(FileFilter):
 			del self.__pending[path]
 		except KeyError: pass
 
+	def getPendingPaths(self):
+		return self.__pending.keys()
+
 	def _checkPending(self):
 		if not self.__pending:
 			return
@@ -225,7 +228,11 @@ class FilterPyinotify(FileFilter):
 	def _delFileWatcher(self, path):
 		try:
 			wdInt = self.__watchFiles.pop(path)
-			wd = self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				wd = self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for file %s", wdInt, path)
+				wd = {wdInt: 1}
 			if wd[wdInt]:
 				logSys.debug("Removed file watcher for %s", path)
 				return True
@@ -249,7 +256,10 @@ class FilterPyinotify(FileFilter):
 		# Remove watches for the directory:
 		try:
 			wdInt = self.__watchDirs.pop(path_dir)
-			self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for directory %s", wdInt, path_dir)
 		except KeyError: # pragma: no cover
 			pass
 			# EnvironmentError is parent of IOError, OSError, etc.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 52187b3d..f8912fef 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -980,6 +980,18 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)
 
+		def test_del_file(self):
+			# test filter reaction by delete watching file:
+			self.file.close()
+			self.waitForTicks(1)
+			# remove file (cause detection of log-rotation)...
+			os.unlink(self.name)
+			# check it was detected (in pending files):
+			self.waitForTicks(2)
+			if hasattr(self.filter, "getPendingPaths"):
+				self.assertTrue(Utils.wait_for(lambda: self.name in self.filter.getPendingPaths(), _maxWaitTime(10)))
+				self.assertEqual(len(self.filter.getPendingPaths()), 1)
+
 		@with_tmpdir
 		def test_move_dir(self, tmp):
 			self.file.close()

From ceff489a46396b561d4a890ae75213e266837a04 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 Oct 2017 14:24:21 +0200
Subject: [PATCH 524/623] amend to a4459765ef438db83a2898ba832ff7acba033e29:
 irrelevant condition removed

---
 fail2ban/server/filterpoll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 5b736a4e..62cf76f0 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -170,7 +170,7 @@ class FilterPoll(FileFilter):
 				else: # pragma: no cover
 					logSys.error("Unable to get stat on %s because of: %s",
 							 filename, e, 
-							 exc_info=e.errno != 2 and logSys.getEffectiveLevel()<=logging.DEBUG)
+							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
 			self.commonError()

From 351abeb4ff77dfca4049e7401553900b46e8db40 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 Oct 2017 13:25:14 +0200
Subject: [PATCH 525/623] prepare release: bump version, update ChangeLog,
 man's and MANIFEST etc.

---
 ChangeLog                |  2 +-
 MANIFEST                 |  6 +++++-
 README.md                |  6 +++---
 fail2ban/version.py      |  2 +-
 man/fail2ban-client.1    |  7 +++++--
 man/fail2ban-regex.1     |  2 +-
 man/fail2ban-server.1    | 10 +++++-----
 man/fail2ban-testcases.1 |  2 +-
 8 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index df0f96be..80e31e14 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -31,7 +31,7 @@ Incompatibility list (compared to v.0.9):
   IPv6-capable now.
 
 
-ver. 0.10.1-dev-1 (2016/??/??) - development edition
+ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
 -----------
 
 ### Fixes
diff --git a/MANIFEST b/MANIFEST
index a6fdae5a..d69ca029 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -120,6 +120,7 @@ config/filter.d/openwebmail.conf
 config/filter.d/oracleims.conf
 config/filter.d/pam-generic.conf
 config/filter.d/perdition.conf
+config/filter.d/phpmyadmin-syslog.conf
 config/filter.d/php-url-fopen.conf
 config/filter.d/portsentry.conf
 config/filter.d/postfix.conf
@@ -148,6 +149,7 @@ config/filter.d/vsftpd.conf
 config/filter.d/webmin-auth.conf
 config/filter.d/wuftpd.conf
 config/filter.d/xinetd-fail.conf
+config/filter.d/zoneminder.conf
 config/jail.conf
 config/paths-arch.conf
 config/paths-common.conf
@@ -309,6 +311,7 @@ fail2ban/tests/files/logs/openwebmail
 fail2ban/tests/files/logs/oracleims
 fail2ban/tests/files/logs/pam-generic
 fail2ban/tests/files/logs/perdition
+fail2ban/tests/files/logs/phpmyadmin-syslog
 fail2ban/tests/files/logs/php-url-fopen
 fail2ban/tests/files/logs/portsentry
 fail2ban/tests/files/logs/postfix
@@ -336,6 +339,7 @@ fail2ban/tests/files/logs/vsftpd
 fail2ban/tests/files/logs/webmin-auth
 fail2ban/tests/files/logs/wuftpd
 fail2ban/tests/files/logs/xinetd-fail
+fail2ban/tests/files/logs/zoneminder
 fail2ban/tests/files/logs/zzz-generic-example
 fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline
 fail2ban/tests/files/testcase01a.log
@@ -363,7 +367,7 @@ files/cacti/fail2ban_stats.sh
 files/cacti/README
 files/debian-initd
 files/fail2ban-logrotate
-files/fail2ban.service
+files/fail2ban.service.in
 files/fail2ban-tmpfiles.conf
 files/fail2ban.upstart
 files/gen_badbots
diff --git a/README.md b/README.md
index bb87b935..f082f808 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.1                 2016/??/??
+                       v0.10.1                 2017/10/12
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
@@ -43,8 +43,8 @@ Optional:
 
 To install, just do:
 
-    tar xvfj fail2ban-0.10.0.tar.bz2
-    cd fail2ban-0.10.0
+    tar xvfj fail2ban-0.10.1.tar.bz2
+    cd fail2ban-0.10.1
     python setup.py install
 
 This will install Fail2Ban into the python library directory. The executable
diff --git a/fail2ban/version.py b/fail2ban/version.py
index f9638bab..bcf9e9b3 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.1.dev1"
+version = "0.10.1"
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 2215d70e..a10fc3e0 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "August 2017" "fail2ban-client v0.10.0" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "October 2017" "fail2ban-client v0.10.1" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.10.0 reads log file that contains password failure report
+Fail2Ban v0.10.1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
@@ -29,6 +29,9 @@ logging level
 \fB\-d\fR
 dump configuration. For debugging
 .TP
+\fB\-\-dp\fR, \fB\-\-dump\-pretty\fR
+dump the configuration using more human readable representation
+.TP
 \fB\-t\fR, \fB\-\-test\fR
 test configuration (can be also specified with start parameters)
 .TP
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index c414bec3..19f42c76 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "August 2017" "fail2ban-regex 0.10.0" "User Commands"
+.TH FAIL2BAN-REGEX "1" "October 2017" "fail2ban-regex 0.10.1" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 5151821e..0201c4d9 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "August 2017" "fail2ban-server v0.10.0" "User Commands"
+.TH FAIL2BAN-SERVER "1" "October 2017" "fail2ban-server v0.10.1" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.10.0 reads log file that contains password failure report
+Fail2Ban v0.10.1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
@@ -23,15 +23,15 @@ pidfile path
 logging level
 .HP
 \fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
-logging target
-.br
-Note. If fail2ban running as systemd-service, for logging to the systemd-journal, the logtarget could be set to STDOUT
 .HP
 \fB\-\-syslogsocket\fR auto|<FILE>
 .TP
 \fB\-d\fR
 dump configuration. For debugging
 .TP
+\fB\-\-dp\fR, \fB\-\-dump\-pretty\fR
+dump the configuration using more human readable representation
+.TP
 \fB\-t\fR, \fB\-\-test\fR
 test configuration (can be also specified with start parameters)
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index e4566005..e1fb7bbb 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "August 2017" "fail2ban-testcases 0.10.0" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "October 2017" "fail2ban-testcases 0.10.1" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS

From 028f32b74b50ae163cbfb1d228d9a8c09ed51ed8 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 12 Oct 2017 14:00:41 +0200
Subject: [PATCH 526/623] bump version (0.10.1 -> 0.10.2.dev1)

---
 ChangeLog                | 10 ++++++++++
 README.md                |  2 +-
 fail2ban/version.py      |  2 +-
 man/fail2ban-client.1    |  4 ++--
 man/fail2ban-regex.1     |  2 +-
 man/fail2ban-server.1    |  4 ++--
 man/fail2ban-testcases.1 |  2 +-
 7 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 80e31e14..7c9319a0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -31,6 +31,16 @@ Incompatibility list (compared to v.0.9):
   IPv6-capable now.
 
 
+ver. 0.10.2-dev-1 (2017/??/??) - development edition
+-----------
+
+### Fixes
+
+### New Features
+
+### Enhancements
+
+
 ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
 -----------
 
diff --git a/README.md b/README.md
index f082f808..0821e691 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.1                 2017/10/12
+                       v0.10.2                 2017/??/??
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
diff --git a/fail2ban/version.py b/fail2ban/version.py
index bcf9e9b3..2fa9b6a5 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.1"
+version = "0.10.2.dev1"
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index a10fc3e0..26e5ee59 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "October 2017" "fail2ban-client v0.10.1" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "October 2017" "fail2ban-client v0.10.2.dev1" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.10.1 reads log file that contains password failure report
+Fail2Ban v0.10.2.dev1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index 19f42c76..a2347b1f 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "October 2017" "fail2ban-regex 0.10.1" "User Commands"
+.TH FAIL2BAN-REGEX "1" "October 2017" "fail2ban-regex 0.10.2.dev1" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index 0201c4d9..cb71e288 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "October 2017" "fail2ban-server v0.10.1" "User Commands"
+.TH FAIL2BAN-SERVER "1" "October 2017" "fail2ban-server v0.10.2.dev1" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.10.1 reads log file that contains password failure report
+Fail2Ban v0.10.2.dev1 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index e1fb7bbb..6feb0162 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "October 2017" "fail2ban-testcases 0.10.1" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "October 2017" "fail2ban-testcases 0.10.2.dev1" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS

From ea1b663f859242b6647ef877ff63dc2579307531 Mon Sep 17 00:00:00 2001
From: Harry Wood <github@onlineanimals.co.uk>
Date: Mon, 16 Oct 2017 01:15:58 +0100
Subject: [PATCH 527/623] typo

spell "positive" (...but also somebody should finish this sentence)
---
 config/filter.d/apache-botsearch.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf
index 5687d405..5e1ee5fa 100644
--- a/config/filter.d/apache-botsearch.conf
+++ b/config/filter.d/apache-botsearch.conf
@@ -3,7 +3,7 @@
 # This filter is aimed at blocking specific URLs that don't exist. This
 # could be a set of URLs places in a Disallow: directive in robots.txt or
 # just some web services that don't exist caused bots are searching for
-# exploitable content. This filter is designed to have a low false postitive
+# exploitable content. This filter is designed to have a low false positive
 # rate due.
 #
 # An alternative to this is the apache-noscript filter which blocks all
@@ -37,4 +37,4 @@ webroot = /var/www/
 
 # DEV Notes:
 #
-# Author: Daniel Black
\ No newline at end of file
+# Author: Daniel Black

From a4f94d2619ebab97b2834f38c7594b44ea509f98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20W=C4=85sikowski?= <lukasz@wasikowski.net>
Date: Sun, 15 Oct 2017 11:10:16 +0200
Subject: [PATCH 528/623] Update pf.conf

Fix comment, because current one won't work:

cat /etc/pf.conf
anchor f2b {
  sshd
}

# service pf reload
Reloading pf rules.
/etc/pf.conf:2: syntax error

New version:

cat /etc/pf.conf
anchor f2b {
  anchor sshd
}

# service pf reload
Reloading pf rules.
---
 config/action.d/pf.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index c70c8e43..a4bd83e0 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -18,8 +18,8 @@
 # also, these rulesets are loaded into (nested) anchors
 # to enable them, add
 #     anchor f2b {
-#        name1
-#        name2
+#        anchor name1
+#        anchor name2
 #        ...
 #     }
 # to your main pf ruleset, where "namei" are the names of the jails

From 8726c9fb0ab3cfd2289718d2092d2c5da81ac726 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 Oct 2017 13:46:29 +0200
Subject: [PATCH 529/623] pf.conf: enclose ports in braces, multiple ports
 expecting this syntax `... any port {http, https}`. Note this would be
 backwards-incompatible change (for the people already enclosing multiports in
 braces in jail.local). closes gh-1915

---
 config/action.d/pf.conf          | 2 +-
 fail2ban/tests/servertestcase.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index a4bd83e0..ea3b9995 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -110,5 +110,5 @@ allports = any
 # Option: multiport
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
-multiport = any port <port>
+multiport = any port {<port>}
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 1d963d0c..270501e3 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1503,7 +1503,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -a f2b/j-w-pf -f-`',
-						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port <port>" | pfctl -a f2b/j-w-pf -f-`',
+						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port {<port>}" | pfctl -a f2b/j-w-pf -f-`',
 					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -a f2b/j-w-pf -f-`',
@@ -1517,12 +1517,12 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-ban':   ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T add 2001:db8::`",),
 					'ip6-unban': ("`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T delete 2001:db8::`",),
 				}),
-				# pf multiport with custom port --
-				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port=http]', {
+				# pf multiport with custom ports --
+				('j-w-pf-mp', 'pf[actiontype=<multiport>][name=%(__name__)s, port="http,https"]', {
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -a f2b/j-w-pf-mp -f-`',
-						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port http" | pfctl -a f2b/j-w-pf-mp -f-`',
+						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port {http,https}" | pfctl -a f2b/j-w-pf-mp -f-`',
 					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf-mp -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -a f2b/j-w-pf-mp -f-`',

From 3c4910a3e219a7d77d8bb99b1987bc78e12898e3 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 Oct 2017 16:06:39 +0200
Subject: [PATCH 530/623] ChangeLog entry + note for possible incompatibility.

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7c9319a0..4e3560b7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -34,7 +34,14 @@ Incompatibility list (compared to v.0.9):
 ver. 0.10.2-dev-1 (2017/??/??) - development edition
 -----------
 
+### Incompatibility list:
+* The configuration for multiport jails using banaction `pf` can be incompatible after upgrade, if
+  ports are enclosed in curly braces `{ }` in the `jail.local` etc. This may cause a double-brackets now.
+
 ### Fixes
+* action.d/pf.conf: 
+  - fixed syntax error in achnor definition (documentation, see gh-1919);
+  - enclose ports in braces for multiport jails (see gh-1925);
 
 ### New Features
 

From a1b863fcf649ba152e2b435f0b14b55ea9dfab1d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 17 Oct 2017 20:12:48 +0200
Subject: [PATCH 531/623] action.d/pf.conf: extended with bulk-unban, command
 `actionflush` in order to flush all bans at once (by stop jail, resp.
 shutdown of fail2ban)

---
 ChangeLog                        |  1 +
 config/action.d/pf.conf          | 13 +++++++++++--
 fail2ban/tests/servertestcase.py |  9 +++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4e3560b7..969fe920 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -46,6 +46,7 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 ### New Features
 
 ### Enhancements
+* action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once.
 
 
 ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index ea3b9995..b3a33cf2 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -16,7 +16,9 @@
 # we don't enable PF automatically; to enable run pfctl -e 
 # or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
 # also, these rulesets are loaded into (nested) anchors
-# to enable them, add
+# to enable them, add as wildcard:
+#     anchor "f2b/*"
+# or using jail names:
 #     anchor f2b {
 #        anchor name1
 #        anchor name2
@@ -37,10 +39,17 @@ actionstart_on_demand = false
 #
 # we only disable PF rules we've installed prior
 actionstop = <pfctl> -sr 2>/dev/null | grep -v <tablename>-<name> | <pfctl> -f-
-             <pfctl> -t <tablename>-<name> -T flush
+             %(actionflush)s
              <pfctl> -t <tablename>-<name> -T kill
 
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = <pfctl> -t <tablename>-<name> -T flush
+
+
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 270501e3..6587c13d 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1505,6 +1505,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -a f2b/j-w-pf -f-`',
 						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port {<port>}" | pfctl -a f2b/j-w-pf -f-`',
 					),
+					'flush': (
+						'`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T flush`',
+					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -a f2b/j-w-pf -f-`',
 						'`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T flush`',
@@ -1524,6 +1527,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -a f2b/j-w-pf-mp -f-`',
 						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port {http,https}" | pfctl -a f2b/j-w-pf-mp -f-`',
 					),
+					'flush': (
+						'`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T flush`',
+					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf-mp -sr 2>/dev/null | grep -v f2b-j-w-pf-mp | pfctl -a f2b/j-w-pf-mp -f-`',
 						'`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T flush`',
@@ -1544,6 +1550,9 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						'`echo "block quick proto tcp from <f2b-j-w-pf-ap> to any" | pfctl -a f2b/j-w-pf-ap -f-`',
 					),
 					'ip6-start': (), # the same as ipv4
+					'flush': (
+						'`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T flush`',
+					),
 					'stop': (
 						'`pfctl -a f2b/j-w-pf-ap -sr 2>/dev/null | grep -v f2b-j-w-pf-ap | pfctl -a f2b/j-w-pf-ap -f-`',
 						'`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T flush`',

From d5d1fe679faee95005a649eadd895b683a622b7b Mon Sep 17 00:00:00 2001
From: Michael Newton <miken32@gmail.com>
Date: Tue, 17 Oct 2017 14:44:23 -0700
Subject: [PATCH 532/623] Remove invalid regex

Resolves #1927
---
 config/filter.d/asterisk.conf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index cd3ea150..e94754c0 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -22,7 +22,6 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
             ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
             ^%(__prefix_line)s%(log_prefix)s (?:Host )?<HOST> (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b)
             ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (?:user|device) [^@]+@<HOST>\S*$
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$

From 3f715e8577000d3cb98998ce9d774083a8647ffa Mon Sep 17 00:00:00 2001
From: Michael Newton <miken32@gmail.com>
Date: Tue, 17 Oct 2017 14:46:11 -0700
Subject: [PATCH 533/623] Remove tests

---
 fail2ban/tests/files/logs/asterisk | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 5e846111..837011e0 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -1,6 +1,4 @@
 # Sample log files for asterisk 
-# failJSON: { "time": "2013-07-25T07:26:43", "match": true , "host": "1.2.3.4" }
-[2013-07-25 07:26:43] NOTICE[26015][C-000006b2] chan_sip.c: Failed to authenticate device 101<sip:101@1.2.3.4>;tag=deadbeef
 # failJSON: { "time": "2012-02-13T17:21:54", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:21:54] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - Wrong password
 # failJSON: { "time": "2012-02-13T17:18:22", "match": true , "host": "1.2.3.4" }
@@ -19,8 +17,6 @@
 [2012-02-13 17:39:20] NOTICE[1638] chan_iax2.c: No registration for peer 'Fail2ban' (from 1.2.3.4)
 # failJSON: { "time": "2012-02-13T17:44:26", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247)
-# failJSON: { "time": "2012-02-13T17:37:07", "match": true , "host": "1.2.3.4" }
-[2012-02-13 17:37:07] NOTICE[1638] chan_sip.c: Failed to authenticate user "Fail2ban" <sip:301@1.2.3.4>;tag=1r698745234
 # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" }
 [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'.
 # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" }

From 894a05b84307704052e79ac142c5f642f76f1302 Mon Sep 17 00:00:00 2001
From: Michael Newton <miken32@gmail.com>
Date: Wed, 18 Oct 2017 09:26:51 -0700
Subject: [PATCH 534/623] Update ChangeLog

---
 ChangeLog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index fce432ed..bc4656b4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -26,7 +26,9 @@ releases.
     - Fixes resources greedy expression (see gh-1790);
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
 * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
-* filter.d/asterisk.conf - fixed failregex AMI Asterisk authentification failed (see gh-1302)
+* filter.d/asterisk.conf
+    - fixed failregex AMI Asterisk authentification failed (see gh-1302)
+    - fixed invalid regex blocking localhost (see gh-1927)
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
     - extended to match pam_authenticate failures with "Permission denied" (gh-1897)

From b6ab0aa83fbfe344ec9cb87806d39d76b6a7a907 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 18 Oct 2017 18:52:12 +0200
Subject: [PATCH 535/623] Update ChangeLog

more detailed entry
---
 ChangeLog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index bc4656b4..70f3d61e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,7 +28,8 @@ releases.
 * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
 * filter.d/asterisk.conf
     - fixed failregex AMI Asterisk authentification failed (see gh-1302)
-    - fixed invalid regex blocking localhost (see gh-1927)
+    - removed invalid (vulnerable) regex blocking IPs using forign data (aus header "from")
+      thus not the IP-address that really originates the request (see gh-1927)
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
     - extended to match pam_authenticate failures with "Permission denied" (gh-1897)

From d81405adbc93a6c6e0dc645814aa83d279855571 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 18 Oct 2017 18:52:55 +0200
Subject: [PATCH 536/623] Update ChangeLog

typo
---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 70f3d61e..f3868e4b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,7 +28,7 @@ releases.
 * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
 * filter.d/asterisk.conf
     - fixed failregex AMI Asterisk authentification failed (see gh-1302)
-    - removed invalid (vulnerable) regex blocking IPs using forign data (aus header "from")
+    - removed invalid (vulnerable) regex blocking IPs using forign data (from header "from")
       thus not the IP-address that really originates the request (see gh-1927)
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)

From 5db497017a63c51ad35b35b2fc5db9108b702de9 Mon Sep 17 00:00:00 2001
From: martin61 <martin61@users.noreply.github.com>
Date: Sun, 15 Oct 2017 11:07:41 +0200
Subject: [PATCH 537/623] add ip6tables.service ipset.service in systemd unit

---
 files/fail2ban.service.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in
index 7114a938..37ae4f07 100644
--- a/files/fail2ban.service.in
+++ b/files/fail2ban.service.in
@@ -1,8 +1,8 @@
 [Unit]
 Description=Fail2Ban Service
 Documentation=man:fail2ban(1)
-After=network.target iptables.service firewalld.service
-PartOf=iptables.service firewalld.service
+After=network.target iptables.service firewalld.service ip6tables.service ipset.service
+PartOf=iptables.service firewalld.service ip6tables.service ipset.service
 
 [Service]
 Type=simple

From e07a8cda07c8b3c3e9ab6ed304ac547fcbff986c Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 27 Oct 2017 15:26:17 +0200
Subject: [PATCH 538/623] Update jail.conf

Documentation of parameters for action blocklist_de, closes gh-1940
---
 config/jail.conf | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 29ad4c96..fb76926d 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -186,10 +186,11 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
 # 
-# See the IMPORTANT note in action.d/blocklist_de.conf for when to
-# use this action. Create a file jail.d/blocklist_de.local containing
-# [Init]
-# blocklist_de_apikey = {api key from registration]
+# See the IMPORTANT note in action.d/blocklist_de.conf for when to use this action.
+# Specify expected parameters in file action.d/blocklist_de.local or if the interpolation
+# `action_blocklist_de` used for the action, set value of `blocklist_de_apikey`
+# in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in 
+# corresponding jail.d/my-jail.local file).
 #
 action_blocklist_de  = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]
 

From b615a9854019589c87d68dd586f45c61f8fe886d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 30 Oct 2017 13:32:52 +0100
Subject: [PATCH 539/623] jail.conf: avoid overwriting of default value of the
 parameter `chain` of several actions (where default chain != INPUT);
 test-cases extended to cover the same logic (use `<known/chain>` instead of
 fix value `INPUT`); Closes gh-1949

---
 config/jail.conf                 |  4 +--
 fail2ban/tests/servertestcase.py | 44 ++++++++++++++++----------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index a9a93eb0..5a6c59e6 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -144,8 +144,8 @@ mta = sendmail
 # Default protocol
 protocol = tcp
 
-# Specify chain where jumps would need to be added in iptables-* actions
-chain = INPUT
+# Specify chain where jumps would need to be added in ban-actions expecting parameter chain
+chain = <known/chain>
 
 # Ports to be banned
 # Usually should be overridden in a particular jail
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 6587c13d..2c293740 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1213,7 +1213,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-multiport --
-				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-mp`",
@@ -1257,7 +1257,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-allports --
-				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-ap`",
@@ -1301,7 +1301,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6 --
-				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
 					'ip4-start': (
 						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
@@ -1339,7 +1339,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-ipset-proto6-allports --
-				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain="<known/chain>"]', {
 					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
 					'ip4-start': (
 						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
@@ -1377,7 +1377,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables --
-				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables`",
@@ -1421,7 +1421,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-new --
-				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`iptables -w -N f2b-j-w-iptables-new`",
@@ -1465,7 +1465,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# iptables-xt_recent-echo --
-				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="INPUT"]', {
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="<known/chain>"]', {
 					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
 					'ip4-start': (
 						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
@@ -1566,23 +1566,23 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip6-unban': ("`pfctl -a f2b/j-w-pf-ap -t f2b-j-w-pf-ap -T delete 2001:db8::`",),
 				}),
 				# firewallcmd-multiport --
-				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-mp', 'firewallcmd-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
-						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
 					), 
 					'ip6-start': (
 						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`",
-						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
 					),
 					'stop': (
-						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`",
-						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`",
 						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`",
 					),
@@ -1606,23 +1606,23 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-allports --
-				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ap', 'firewallcmd-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': (' ipv4 ', 'icmp-port-unreachable'), 'ip6': (' ipv6 ', 'icmp6-port-unreachable'),
 					'ip4-start': (
 						"`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
-						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -j f2b-j-w-fwcmd-ap`",
 					), 
 					'ip6-start': (
 						"`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-ap 1000 -j RETURN`",
-						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -j f2b-j-w-fwcmd-ap`",
 					),
 					'stop': (
-						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -j f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-ap`",
-						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -j f2b-j-w-fwcmd-ap`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -j f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-ap`",
 						"`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-ap`",
 					),
@@ -1646,21 +1646,21 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					),					
 				}),
 				# firewallcmd-ipset --
-				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="INPUT"]', {
+				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
 					'ip4-start': (
 						"`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`",
-						"`firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 					), 
 					'ip6-start': (
 						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`",
-						"`firewall-cmd --direct --add-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
 					'stop': (
-						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-fwcmd-ipset`",
 						"`ipset destroy f2b-j-w-fwcmd-ipset`",
-						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 						"`ipset flush f2b-j-w-fwcmd-ipset6`",
 						"`ipset destroy f2b-j-w-fwcmd-ipset6`",
 					),

From 18d2761dc0738b86f4a241b8bb9a2aa4b8f6ec07 Mon Sep 17 00:00:00 2001
From: Stuart Cardall <developer@it-offshore.co.uk>
Date: Sat, 28 Oct 2017 16:14:03 +0000
Subject: [PATCH 540/623] gentoo-initd: add descriptions

add descriptions to stop syslog errors for extra_started_commands when running:

rc-service ipset describe

Oct 28 15:13:30 xxxx daemon.warn /etc/init.d/fail2ban[26446]: ^[[1m^[[36mreload^[[m: no description
Oct 28 15:13:30 xxxx daemon.warn /etc/init.d/fail2ban[26447]: ^[[1m^[[36mshowlog^[[m: no description
---
 files/gentoo-initd | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/files/gentoo-initd b/files/gentoo-initd
index c4d8675c..b47e2c47 100755
--- a/files/gentoo-initd
+++ b/files/gentoo-initd
@@ -18,6 +18,9 @@
 # Author: Sireyessire, Cyril Jaquier
 #
 
+description="Daemon to ban hosts that cause multiple authentication errors"
+description_reload="reload configuration"
+description_showlog="show fail2ban logs"
 extra_started_commands="reload showlog"
 
 FAIL2BAN="/usr/bin/fail2ban-client ${FAIL2BAN_OPTIONS}"

From 4a2fc8b7e85554ac819e30b48d300550a9cced10 Mon Sep 17 00:00:00 2001
From: Jeff Potter <jpotter@users.noreply.github.com>
Date: Sun, 25 Jun 2017 19:05:13 -0400
Subject: [PATCH 541/623] Include imap (port 143) in courier-auth ports

imap was missing from the list of ports, preventing fail2ban from blocking connections on standard IMAP port 143.
---
 config/jail.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/jail.conf b/config/jail.conf
index 5a6c59e6..c9fb9475 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -622,7 +622,7 @@ logpath = /opt/kerio/mailserver/store/logs/security.log
 
 [courier-auth]
 
-port     = smtp,465,submission,imap3,imaps,pop3,pop3s
+port     = smtp,465,submission,imap,imap3,imaps,pop3,pop3s
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 

From 9876dd44f93a0b012108ea1aceeccee21be03cb9 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 3 Nov 2017 13:43:33 +0100
Subject: [PATCH 542/623] replace port imap3 with imap everywhere, since imap3
 is not a standard port and old rarely (if ever) used and missing on some
 systems

(see gh-1942)
---
 config/jail.conf | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index c9fb9475..41cb3529 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -622,7 +622,7 @@ logpath = /opt/kerio/mailserver/store/logs/security.log
 
 [courier-auth]
 
-port     = smtp,465,submission,imap,imap3,imaps,pop3,pop3s
+port     = smtp,465,submission,imap,imaps,pop3,pop3s
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
 
@@ -630,7 +630,7 @@ backend  = %(syslog_backend)s
 [postfix-sasl]
 
 filter   = postfix[mode=auth]
-port     = smtp,465,submission,imap3,imaps,pop3,pop3s
+port     = smtp,465,submission,imap,imaps,pop3,pop3s
 # You might consider monitoring /var/log/mail.warn instead if you are
 # running postfix since it would provide the same log lines at the
 # "warn" level but overall at the smaller filesize.
@@ -640,27 +640,27 @@ backend  = %(postfix_backend)s
 
 [perdition]
 
-port   = imap3,imaps,pop3,pop3s
+port   = imap,imaps,pop3,pop3s
 logpath = %(syslog_mail)s
 backend = %(syslog_backend)s
 
 
 [squirrelmail]
 
-port = smtp,465,submission,imap2,imap3,imaps,pop3,pop3s,http,https,socks
+port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks
 logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log
 
 
 [cyrus-imap]
 
-port   = imap3,imaps
+port   = imap,imaps
 logpath = %(syslog_mail)s
 backend = %(syslog_backend)s
 
 
 [uwimap-auth]
 
-port   = imap3,imaps
+port   = imap,imaps
 logpath = %(syslog_mail)s
 backend = %(syslog_backend)s
 

From 8b26fd27780ef4a211dfa312e99e6f69bf181c16 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 3 Nov 2017 13:54:40 +0100
Subject: [PATCH 543/623] Update ChangeLog

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f8995d3d..3ddd159c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -39,6 +39,8 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   ports are enclosed in curly braces `{ }` in the `jail.local` etc. This may cause a double-brackets now.
 
 ### Fixes
+* jail.conf: port `imap3` replace with `imap` everywhere, since imap3 is not a standard port and old rarely 
+  (if ever) used and missing on some systems (e. g. debian stretch), see gh-1942.
 * action.d/pf.conf: 
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);

From 4d10c615c4ca95b1e29f3e2a394ad7922827a2f4 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 3 Nov 2017 14:05:17 +0100
Subject: [PATCH 544/623] Update ChangeLog

typo
---
 ChangeLog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3ddd159c..c6115409 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -39,8 +39,8 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   ports are enclosed in curly braces `{ }` in the `jail.local` etc. This may cause a double-brackets now.
 
 ### Fixes
-* jail.conf: port `imap3` replace with `imap` everywhere, since imap3 is not a standard port and old rarely 
-  (if ever) used and missing on some systems (e. g. debian stretch), see gh-1942.
+* jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely 
+  (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
 * action.d/pf.conf: 
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);

From eba68a8f377795ca151d7ebd9199107a538cbb7a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 3 Nov 2017 11:31:36 +0100
Subject: [PATCH 545/623] config/paths-common.conf: Added initial values for
 `syslog_authpriv`, `syslog_mail` in order to avoid errors while
 parsing/interpolating configuration; Note the systemd-backend does not need
 the logpath at all; Some defaults normalized (minimized configs, don't need
 to overwrite values in distribution-related path if equal).

---
 config/paths-common.conf   | 33 ++++++++++++++++++++++++---------
 config/paths-debian.conf   | 11 -----------
 config/paths-fedora.conf   |  9 ---------
 config/paths-freebsd.conf  |  9 ---------
 config/paths-opensuse.conf | 12 ------------
 config/paths-osx.conf      |  8 ++++----
 6 files changed, 28 insertions(+), 54 deletions(-)

diff --git a/config/paths-common.conf b/config/paths-common.conf
index 51323d6b..7383cafe 100644
--- a/config/paths-common.conf
+++ b/config/paths-common.conf
@@ -9,21 +9,36 @@ after  = paths-overrides.local
 
 default_backend = %(default/backend)s
 
+# Initial common values (to overwrite in path-<distribution>.conf)...
+# There is no sensible generic defaults for syslog log targets, thus
+# leaving them empty here (resp. set to mostly used variant) in order 
+# to avoid errors while parsing/interpolating configs.
+#
+# Note systemd-backend does not need the logpath at all.
+#
+syslog_local0 = /var/log/messages
+
+syslog_authpriv = /var/log/auth.log
+syslog_daemon  = %(syslog_local0)s
+syslog_ftp = %(syslog_local0)s
+syslog_mail =
+syslog_mail_warn =
+syslog_user = %(syslog_local0)s
+
+# Set the default syslog backend target to default_backend
+syslog_backend = %(default_backend)s
+
+# Default values for several jails:
+
 sshd_log = %(syslog_authpriv)s
 sshd_backend = %(default_backend)s
 
 dropbear_log = %(syslog_authpriv)s
 dropbear_backend = %(default_backend)s
 
-# There is no sensible generic defaults for syslog log targets, thus
-# leaving them empty here so that no errors while parsing/interpolating configs
-syslog_daemon =
-syslog_ftp =
-syslog_local0 =
-syslog_mail_warn =
-syslog_user =
-# Set the default syslog backend target to default_backend
-syslog_backend = %(default_backend)s
+apache_error_log = /var/log/apache2/*error.log
+
+apache_access_log = /var/log/apache2/*access.log
 
 # from /etc/audit/auditd.conf
 auditd_log = /var/log/audit/audit.log
diff --git a/config/paths-debian.conf b/config/paths-debian.conf
index 1ca4d242..e096f972 100644
--- a/config/paths-debian.conf
+++ b/config/paths-debian.conf
@@ -15,23 +15,12 @@ syslog_mail = /var/log/mail.log
 # syslog_mail_warn = /var/log/mail.warn
 syslog_mail_warn = %(syslog_mail)s
 
-syslog_authpriv = /var/log/auth.log
-
-# syslog_auth = /var/log/auth.log
-#
 syslog_user  =  /var/log/user.log
 
 syslog_ftp = /var/log/syslog
 
 syslog_daemon =  /var/log/daemon.log
 
-syslog_local0 =  /var/log/messages
-
-
-apache_error_log = /var/log/apache2/*error.log
-
-apache_access_log = /var/log/apache2/*access.log
-
 exim_main_log = /var/log/exim4/mainlog
 
 # was in debian squeezy but not in wheezy
diff --git a/config/paths-fedora.conf b/config/paths-fedora.conf
index d13645b1..3d637e1f 100644
--- a/config/paths-fedora.conf
+++ b/config/paths-fedora.conf
@@ -15,15 +15,6 @@ syslog_mail_warn = /var/log/maillog
 
 syslog_authpriv = /var/log/secure
 
-syslog_user =  /var/log/messages
-
-syslog_ftp  = /var/log/messages
-
-syslog_daemon  = /var/log/messages
-
-syslog_local0  = /var/log/messages
-
-
 apache_error_log = /var/log/httpd/*error_log
 
 apache_access_log = /var/log/httpd/*access_log
diff --git a/config/paths-freebsd.conf b/config/paths-freebsd.conf
index 91b23636..550ee887 100644
--- a/config/paths-freebsd.conf
+++ b/config/paths-freebsd.conf
@@ -15,19 +15,10 @@ syslog_mail = /var/log/maillog
 
 syslog_mail_warn = /var/log/maillog
 
-syslog_authpriv = /var/log/auth.log
-
 # note - is only ftp.info - if notice /var/log/messages may be needed
 syslog_ftp = /var/log/xferlog
 
-syslog_daemon = /var/log/messages
-
-syslog_local0 = /var/log/messages
-
 # Linux things
-# we fake to avoid parse error in startups
-
-auditd_log = /dev/null
 
 # http://svnweb.freebsd.org/ports/head/www/apache24/files/patch-docs__conf__extra__httpd-ssl.conf.in?view=markup
 # http://svnweb.freebsd.org/ports/head/www/apache22/files/patch-docs__conf__extra__httpd-ssl.conf.in?view=markup
diff --git a/config/paths-opensuse.conf b/config/paths-opensuse.conf
index 227a5e98..f1d70ce7 100644
--- a/config/paths-opensuse.conf
+++ b/config/paths-opensuse.conf
@@ -9,24 +9,12 @@ after  = paths-overrides.local
 
 [DEFAULT]
 
-syslog_local0  = /var/log/messages
-
 syslog_mail = /var/log/mail
 
 syslog_mail_warn = %(syslog_mail)s
 
 syslog_authpriv = %(syslog_local0)s
 
-syslog_user =  %(syslog_local0)s
-
-syslog_ftp  = %(syslog_local0)s
-
-syslog_daemon  = %(syslog_local0)s
-
-apache_error_log = /var/log/apache2/*error_log
-
-apache_access_log = /var/log/apache2/*access_log
-
 pureftpd_log = %(syslog_local0)s
 
 exim_main_log = /var/log/exim/main.log
diff --git a/config/paths-osx.conf b/config/paths-osx.conf
index d1b99b38..2fffd65a 100644
--- a/config/paths-osx.conf
+++ b/config/paths-osx.conf
@@ -17,11 +17,11 @@ syslog_mail_warn = /var/log/mail.warn
 syslog_authpriv = /var/log/secure.log
 #syslog_auth = 
 
-#syslog_user = 
+syslog_user = 
 
-#syslog_ftp = 
+syslog_ftp = 
 
-#syslog_daemon = 
+syslog_daemon = 
 
-#syslog_local0 =
+syslog_local0 =
 

From ee80c524301d498f9cbf3ab359bf620f46808057 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 3 Nov 2017 13:13:41 +0100
Subject: [PATCH 546/623] Update ChangeLog

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c6115409..757774a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -41,6 +41,8 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 ### Fixes
 * jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely 
   (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
+* config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf)
+  in order to avoid errors while interpolating (e. g. starting with systemd-backend), see gh-1955.
 * action.d/pf.conf: 
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);

From 159957ab883efc151a2dddf53197153ba41576c2 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Nov 2017 22:21:42 +0100
Subject: [PATCH 547/623] filter.d/sshd.conf: extended failregex for modes
 "extra"/"aggressive": now finds all possible (also future) forms of "no
 matching (cipher|mac|MAC|compression method|key exchange method|host key
 type) found", see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors;
 obsolete (multi-line buffered) variant extended also.

Closes gh-1943, gh-1944
---
 ChangeLog                                          |  5 ++++-
 config/filter.d/sshd.conf                          |  9 +++++++--
 .../filter.d/zzz-sshd-obsolete-multiline.conf      |  9 +++++++--
 fail2ban/tests/files/logs/sshd                     | 14 ++++++++++++++
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 757774a6..44834880 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -43,9 +43,12 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
 * config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf)
   in order to avoid errors while interpolating (e. g. starting with systemd-backend), see gh-1955.
-* action.d/pf.conf: 
+* `action.d/pf.conf`: 
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);
+* `filter.d/sshd.conf`: extended failregex for modes "extra"/"aggressive": now finds all possible (also future)
+  forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", 
+  see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
 
 ### New Features
 
diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 95915fcc..0f9a32ed 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -24,6 +24,10 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)?
 __suff = (?: \[preauth\])?\s*
 __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
 
+# for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
+# see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
+__alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
+
 [Definition]
 
 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
@@ -55,8 +59,9 @@ mdre-ddos = ^Did not receive identification string from <HOST>%(__suff)s$
             ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer%(__suff)s
 
 mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
-            ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
-            ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+            ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
+            ^Unable to negotiate a <__alg_match>%(__suff)s$
+            ^no matching <__alg_match> found:
 
 mdre-aggressive = %(mdre-ddos)s
                   %(mdre-extra)s
diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
index d6eecd4b..5a3c44e3 100644
--- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
@@ -23,6 +23,10 @@ __prefix_line_sl = %(__prefix_line)s%(__pref)s
 __prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s
 __prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
 
+# for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
+# see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
+__alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
+
 [Definition]
 
 cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
@@ -50,8 +54,9 @@ mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HO
              ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
 
 mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
-             ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching (?:cipher|key exchange method) found.
-             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$
+             ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
+             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match>%(__suff)s$
+             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found:
 
 mdre-aggressive = %(mdre-ddos)s
                   %(mdre-extra)s
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index b9559359..ce0a2426 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -243,3 +243,17 @@ Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.16
 Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
 # failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" }
 Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
+
+# gh-1943 (previous OpenSSH log-format)
+# failJSON: { "match": false }
+Oct 25 19:37:08 hostname sshd[22477]: Connection from 192.0.2.1 port 31309 on 192.0.2.8 port 22
+# failJSON: { "time": "2004-10-25T19:37:09", "match": true , "host": "192.0.2.1", "desc": "No matching mac found" }
+Oct 25 19:37:09 hostname sshd[22477]: fatal: no matching mac found: client hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx server hmac-xxx,hmac-xxx,umac-xxx,hmac-xxx,hmac-xxx,umac-xxx [preauth]
+
+# gh-1944 (newest OpenSSH log-format)
+# failJSON: { "time": "2004-10-26T15:30:40", "match": true , "host": "192.0.2.2", "desc": "No matching MAC found" }
+Oct 26 15:30:40 localhost sshd[14737]: Unable to negotiate with 192.0.2.2 port 50404: no matching MAC found. Their offer: hmac-sha1,hmac-sha1-96,hmac-md5,hmac-md5-96,hmac-ripemd160,hmac-ripemd160@openssh.com [preauth]
+# failJSON: { "time": "2004-11-26T13:03:38", "match": true , "host": "192.0.2.4", "desc": "No matching everything ... found." }
+Nov 26 13:03:38 srv sshd[14737]: Unable to negotiate with 192.0.2.4 port 50404: no matching host key type found. Their offer: ssh-dss
+# failJSON: { "time": "2004-11-26T13:03:39", "match": true , "host": "192.0.2.5", "desc": "No matching everything ... found." }
+Nov 26 13:03:39 srv sshd[14738]: fatal: Unable to negotiate with 192.0.2.5 port 55555: no matching everything new here found. Their offer: ...

From 5708b8b90eaec82a40a94a3a844674103b2345b1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 23 Nov 2017 22:42:51 +0100
Subject: [PATCH 548/623] fixed test-cases covering dns2ip (IP of www.epfl.ch
 changed)

---
 fail2ban/tests/filtertestcase.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index e9971a06..74bcb3ce 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -316,14 +316,14 @@ class IgnoreIPDNS(IgnoreIP):
 
 	def testIgnoreIPDNSOK(self):
 		self.filter.addIgnoreIP("www.epfl.ch")
-		self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12"))
+		self.assertTrue(self.filter.inIgnoreIPList("128.178.222.69"))
 
 	def testIgnoreIPDNSNOK(self):
 		# Test DNS
 		self.filter.addIgnoreIP("www.epfl.ch")
-		self.assertFalse(self.filter.inIgnoreIPList("127.177.50.10"))
-		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11"))
-		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13"))
+		self.assertFalse(self.filter.inIgnoreIPList("127.178.222.69"))
+		self.assertFalse(self.filter.inIgnoreIPList("128.178.222.68"))
+		self.assertFalse(self.filter.inIgnoreIPList("128.178.222.70"))
 
 
 class LogFile(LogCaptureTestCase):

From eac80966c503b0bc940c119d9a0adafb9ccf50d4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Nov 2017 12:54:45 +0100
Subject: [PATCH 549/623] Fix scripts-root within `fail2ban.service` (relative
 install root-base directory). This is amend for
 e3b061e94b54067525c5e7f2ac716d1c838c9f20. Closes gh-1964

---
 setup.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 11748778..5dbd5b1a 100755
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,13 @@ class install_scripts_f2b(install_scripts):
 
 	def update_scripts(self, dry_run=False):
 		buildroot = os.path.dirname(self.build_dir)
-		print('Creating %s/fail2ban.service (from fail2ban.service.in): @BINDIR@ -> %s' % (buildroot, self.install_dir))
+		try:
+			root = self.distribution.command_options['install']['root'][1]
+			if self.install_dir.startswith(root):
+				install_dir = self.install_dir[len(root):]
+		except: # pragma: no cover
+			print('WARNING: Cannot find root-base option, check the bin-path to fail2ban-scripts in "fail2ban.service".')
+		print('Creating %s/fail2ban.service (from fail2ban.service.in): @BINDIR@ -> %s' % (buildroot, install_dir))
 		with open(os.path.join(source_dir, 'files/fail2ban.service.in'), 'r') as fn:
 			lines = fn.readlines()
 		fn = None
@@ -96,7 +102,7 @@ class install_scripts_f2b(install_scripts):
 			fn = open(os.path.join(buildroot, 'fail2ban.service'), 'w')
 		try:
 			for ln in lines:
-				ln = re.sub(r'@BINDIR@', lambda v: self.install_dir, ln)
+				ln = re.sub(r'@BINDIR@', lambda v: install_dir, ln)
 				if dry_run:
 					sys.stdout.write(' | ' + ln)
 					continue

From fa007bfa7cdc27c91727336f8ff64036f71ba379 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Nov 2017 12:57:55 +0100
Subject: [PATCH 550/623] remove build folder, if created through setup-process
 in test

---
 fail2ban/tests/misctestcase.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py
index 373a7178..fda1a5cd 100644
--- a/fail2ban/tests/misctestcase.py
+++ b/fail2ban/tests/misctestcase.py
@@ -121,6 +121,7 @@ class SetupTest(unittest.TestCase):
 		if not self.setup:
 			return			  # if verbose skip didn't work out
 		tmp = tempfile.mkdtemp()
+		remove_build = not os.path.exists('build')
 		# suppress stdout (and stderr) if not heavydebug
 		supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
@@ -178,6 +179,8 @@ class SetupTest(unittest.TestCase):
 			# remove build directory
 			os.system("%s %s clean --all%s"
 					  % (sys.executable, self.setup, (supdbgout + ' 2>&1') if supdbgout else ''))
+			if remove_build and os.path.exists('build'):
+				shutil.rmtree('build')
 
 
 class TestsUtilsTest(LogCaptureTestCase):

From 7bf5980defad39c1928209c7636269f712858f44 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Nov 2017 13:20:19 +0100
Subject: [PATCH 551/623] no root option if testing within virtualenv (fixed
 now).

---
 setup.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 5dbd5b1a..8da29268 100755
--- a/setup.py
+++ b/setup.py
@@ -88,10 +88,12 @@ class install_scripts_f2b(install_scripts):
 
 	def update_scripts(self, dry_run=False):
 		buildroot = os.path.dirname(self.build_dir)
+		install_dir = self.install_dir
 		try:
+			# remove root-base from install scripts path:
 			root = self.distribution.command_options['install']['root'][1]
-			if self.install_dir.startswith(root):
-				install_dir = self.install_dir[len(root):]
+			if install_dir.startswith(root):
+				install_dir = install_dir[len(root):]
 		except: # pragma: no cover
 			print('WARNING: Cannot find root-base option, check the bin-path to fail2ban-scripts in "fail2ban.service".')
 		print('Creating %s/fail2ban.service (from fail2ban.service.in): @BINDIR@ -> %s' % (buildroot, install_dir))

From 100b531affe5bb3e6b3ee8f1aa59d3a1f5a74aeb Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 24 Nov 2017 13:33:20 +0100
Subject: [PATCH 552/623] travis: add build for python 3.7-dev and switch to
 newest pypy3.3 in travis

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d2b60ed1..0c611a95 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,9 +12,10 @@ python:
   - 3.4
   - 3.5
   - 3.6
+  - 3.7-dev
   # disabled since setuptools dropped support for Python 3.0 - 3.2
   # - pypy3
-  - pypy3.3-5.2-alpha1
+  - pypy3.3-5.5-alpha
 before_install:
   - echo "running under $TRAVIS_PYTHON_VERSION"
   - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi

From f31195a4fc1acc346268220e02cd79d572dea16f Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 26 Nov 2017 23:03:29 +0100
Subject: [PATCH 553/623] added new logtarget "SYSOUT" to log from fail2ban
 working in foreground as systemd-service (in opposite to "STDOUT" don't log
 time-stamps).

---
 config/fail2ban.conf               |  2 +-
 fail2ban/client/fail2bancmdline.py |  2 +-
 fail2ban/client/fail2banserver.py  |  3 ++-
 fail2ban/helpers.py                |  6 ++++--
 fail2ban/server/server.py          | 15 +++++++++------
 files/bash-completion              |  2 +-
 files/fail2ban.service.in          |  4 ++--
 man/fail2ban-client.1              |  5 +++--
 man/fail2ban-server.1              |  5 +++--
 9 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/config/fail2ban.conf b/config/fail2ban.conf
index 7c001155..52e47187 100644
--- a/config/fail2ban.conf
+++ b/config/fail2ban.conf
@@ -30,7 +30,7 @@ loglevel = INFO
 #         using logrotate -- also adjust or disable rotation in the
 #         corresponding configuration file
 #         (e.g. /etc/logrotate.d/fail2ban on Debian systems)
-# Values: [ STDOUT | STDERR | SYSLOG | FILE ]  Default: STDERR
+# Values: [ STDOUT | STDERR | SYSLOG | SYSOUT | FILE ]  Default: STDERR
 #
 logtarget = /var/log/fail2ban.log
 
diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py
index 401aa9b6..269fa174 100644
--- a/fail2ban/client/fail2bancmdline.py
+++ b/fail2ban/client/fail2bancmdline.py
@@ -99,7 +99,7 @@ class Fail2banCmdLine():
 		output("    -s <FILE>               socket path")
 		output("    -p <FILE>               pidfile path")
 		output("    --loglevel <LEVEL>      logging level")
-		output("    --logtarget <FILE>|STDOUT|STDERR|SYSLOG")
+		output("    --logtarget <TARGET>    logging target, use file-name or stdout, stderr, syslog or sysout.")
 		output("    --syslogsocket auto|<FILE>")
 		output("    -d                      dump configuration. For debugging")
 		output("    --dp, --dump-pretty     dump the configuration using more human readable representation")
diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py
index 006a02cf..2dcaddf7 100644
--- a/fail2ban/client/fail2banserver.py
+++ b/fail2ban/client/fail2banserver.py
@@ -210,7 +210,8 @@ class Fail2banServer(Fail2banCmdLine):
 					if server: # pragma: no cover
 						server.quit()
 					exit(-1)
-				logSys.debug('Starting server done')
+				if background:
+					logSys.debug('Starting server done')
 
 		except Exception as e:
 			if self._conf["verbose"] > 1:
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 7eaa59f8..5b027b32 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -143,7 +143,7 @@ def str2LogLevel(value):
 		raise ValueError("Invalid log level %r" % value)
 	return ll
 
-def getVerbosityFormat(verbosity, fmt=' %(message)s'):
+def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True):
 	"""Custom log format for the verbose runs
 	"""
 	if verbosity > 1: # pragma: no cover
@@ -152,7 +152,9 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s'):
 		if verbosity > 2:
 			fmt = ' +%(relativeCreated)5d %(thread)X %(name)-25.25s %(levelname)-5.5s' + fmt
 		else:
-			fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt
+			fmt = ' %(thread)X %(levelname)-5.5s' + fmt
+			if addtime:
+				fmt = ' %(asctime)-15s' + fmt
 	return fmt
 
 
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index e3b22c44..ce28e3f9 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -27,7 +27,6 @@ __license__ = "GPL"
 import threading
 from threading import Lock, RLock
 import logging
-import logging.handlers
 import os
 import signal
 import stat
@@ -561,10 +560,8 @@ class Server:
 				self.__logTarget = target
 				return True
 			# set a format which is simpler for console use
-			fmt = "%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
+			fmt = "%(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
 			if systarget == "SYSLOG":
-				# Syslog daemons already add date to the message.
-				fmt = "%(name)s[%(process)d]: %(levelname)s %(message)s"
 				facility = logging.handlers.SysLogHandler.LOG_DAEMON
 				if self.__syslogSocket == "auto":
 					import platform
@@ -581,7 +578,7 @@ class Server:
 						"Syslog socket file: %s does not exists"
 						" or is not a socket" % self.__syslogSocket)
 					return False
-			elif systarget == "STDOUT":
+			elif systarget in ("STDOUT", "SYSOUT"):
 				hdlr = logging.StreamHandler(sys.stdout)
 			elif systarget == "STDERR":
 				hdlr = logging.StreamHandler(sys.stderr)
@@ -615,8 +612,14 @@ class Server:
 			if logger.getEffectiveLevel() <= logging.DEBUG: # pragma: no cover
 				if self.__verbose is None:
 					self.__verbose = logging.DEBUG - logger.getEffectiveLevel() + 1
+			# If handler don't already add date to the message:
+			addtime = systarget not in ("SYSLOG", "SYSOUT")
+			# verbose log-format:
 			if self.__verbose is not None and self.__verbose > 2: # pragma: no cover
-				fmt = getVerbosityFormat(self.__verbose-1)
+				fmt = getVerbosityFormat(self.__verbose-1,
+					addtime=addtime)
+			elif addtime:
+				fmt = "%(asctime)s " + fmt
 			# tell the handler to use this format
 			hdlr.setFormatter(logging.Formatter(fmt))
 			logger.addHandler(hdlr)
diff --git a/files/bash-completion b/files/bash-completion
index 36e0cbba..b8887001 100644
--- a/files/bash-completion
+++ b/files/bash-completion
@@ -108,7 +108,7 @@ _fail2ban () {
                     ;;
                 logtarget)
                     if [[ "$cmd" == "set" ]];then
-                        COMPREPLY=( $( compgen -W "STDOUT STDERR SYSLOG" -- "$cur" ) )
+                        COMPREPLY=( $( compgen -W "STDOUT STDERR SYSLOG SYSOUT" -- "$cur" ) )
                         _filedir # And files
                     fi
                     return 0
diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in
index 37ae4f07..24dcb51e 100644
--- a/files/fail2ban.service.in
+++ b/files/fail2ban.service.in
@@ -8,8 +8,8 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service
 Type=simple
 ExecStartPre=/bin/mkdir -p /var/run/fail2ban
 ExecStart=@BINDIR@/fail2ban-server -xf start
-# if should be logged in systemd journal, use following line or set logtarget to stdout in fail2ban.local
-# ExecStart=@BINDIR@/fail2ban-server -xf --logtarget=stdout start
+# if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local
+# ExecStart=@BINDIR@/fail2ban-server -xf --logtarget=sysout start
 ExecStop=@BINDIR@/fail2ban-client stop
 ExecReload=@BINDIR@/fail2ban-client reload
 PIDFile=/var/run/fail2ban/fail2ban.pid
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 26e5ee59..6e44b387 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -21,8 +21,9 @@ pidfile path
 .TP
 \fB\-\-loglevel\fR <LEVEL>
 logging level
-.HP
-\fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
+.TP
+\fB\-\-logtarget\fR <TARGET>
+logging target, use file\-name or stdout, stderr, syslog or sysout.
 .HP
 \fB\-\-syslogsocket\fR auto|<FILE>
 .TP
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index cb71e288..aca3eeb0 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -21,8 +21,9 @@ pidfile path
 .TP
 \fB\-\-loglevel\fR <LEVEL>
 logging level
-.HP
-\fB\-\-logtarget\fR <FILE>|STDOUT|STDERR|SYSLOG
+.TP
+\fB\-\-logtarget\fR <TARGET>
+logging target, use file\-name or stdout, stderr, syslog or sysout.
 .HP
 \fB\-\-syslogsocket\fR auto|<FILE>
 .TP

From af0f7e93ce3f9b2879e43f544b01464a3448c211 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 26 Nov 2017 23:06:35 +0100
Subject: [PATCH 554/623] better handling by start/stop of server in foreground
 mode; don't call logging.shutdown because part of exit in fail2bancmdline.

---
 fail2ban/server/server.py | 51 ++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index ce28e3f9..9cb6f320 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -151,24 +151,13 @@ class Server:
 			self.__asyncServer.start(sock, force)
 		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)
-		# Removes the PID file.
-		try:
-			logSys.debug("Remove PID file %s", pidfile)
-			os.remove(pidfile)
-		except (OSError, IOError) as e: # pragma: no cover
-			logSys.error("Unable to remove PID file: %s", e)
-		logSys.info("Exiting Fail2ban")
-	
-	def quit(self):
-		# Stop communication first because if jail's unban action
-		# tries to communicate via fail2ban-client we get a lockup
-		# among threads.  So the simplest resolution is to stop all
-		# communications first (which should be ok anyways since we
-		# are exiting)
-		# See https://github.com/fail2ban/fail2ban/issues/7
-		if self.__asyncServer is not None:
-			self.__asyncServer.stop()
-			self.__asyncServer = None
+
+		logSys.info("Shutdown in progress...")
+
+		# Restore default signal handlers:
+		if _thread_name() == '_MainThread':
+			for s, sh in self.__prev_signals.iteritems():
+				signal.signal(s, sh)
 
 		# Now stop all the jails
 		self.stopAllJail()
@@ -179,18 +168,26 @@ class Server:
 			self.__db.close()
 			self.__db = None
 
-		# Only now shutdown the logging.
-		if self.__logTarget is not None:
-			with self.__loggingLock:
-				logging.shutdown()
-
-		# Restore default signal handlers:
-		if _thread_name() == '_MainThread':
-			for s, sh in self.__prev_signals.iteritems():
-				signal.signal(s, sh)
+		# Removes the PID file.
+		try:
+			logSys.debug("Remove PID file %s", pidfile)
+			os.remove(pidfile)
+		except (OSError, IOError) as e: # pragma: no cover
+			logSys.error("Unable to remove PID file: %s", e)
+		logSys.info("Exiting Fail2ban")
 
+	def quit(self):
 		# Prevent to call quit twice:
 		self.quit = lambda: False
+		# Stop communication first because if jail's unban action
+		# tries to communicate via fail2ban-client we get a lockup
+		# among threads.  So the simplest resolution is to stop all
+		# communications first (which should be ok anyways since we
+		# are exiting)
+		# See https://github.com/fail2ban/fail2ban/issues/7
+		if self.__asyncServer is not None:
+			self.__asyncServer.stop()
+			self.__asyncServer = None
 
 	def addJail(self, name, backend):
 		addflg = True

From 6db9ae857468835e38426290cb6e194f813bf0b4 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Sun, 26 Nov 2017 23:35:11 +0100
Subject: [PATCH 555/623] ChangeLog updated

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 44834880..9abafd79 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -39,6 +39,8 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   ports are enclosed in curly braces `{ }` in the `jail.local` etc. This may cause a double-brackets now.
 
 ### Fixes
+* Fixed logging to systemd-journal: new logtarget value SYSOUT can be used instead of STDOUT, to avoid 
+  write of the time-stamp, if logging to systemd-journal from foreground mode (gh-1876)
 * jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely 
   (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
 * config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf)

From 76f28658835aac152d4f850a7f43930b871e3b4c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Nov 2017 13:42:41 +0100
Subject: [PATCH 556/623] implemented new action
 "action.d/nginx-block-map.conf", used in order to ban not IP-related tickets
 via nginx (session blacklisting in nginx-location with map-file);

---
 config/action.d/nginx-block-map.conf     | 108 ++++++++++++++++++
 fail2ban/tests/fail2banclienttestcase.py | 133 +++++++++++++++++++----
 2 files changed, 218 insertions(+), 23 deletions(-)
 create mode 100644 config/action.d/nginx-block-map.conf

diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf
new file mode 100644
index 00000000..33c15f9c
--- /dev/null
+++ b/config/action.d/nginx-block-map.conf
@@ -0,0 +1,108 @@
+# Fail2Ban configuration file for black-listing via nginx
+#
+# Author: Serg G. Brester (aka sebres)
+#
+# To use 'nginx-block-map' action you should define some special blocks in your nginx configuration,
+# and use it hereafter in your locations (to notify fail2ban by failure, resp. nginx by ban).
+#
+# Example (argument "token_id" resp. cookie "session_id" used here as unique identifier for user):
+#
+#   http {
+#     ...
+#     # maps to check user is blacklisted (banned in f2b):
+#     #map $arg_token_id      $blck_lst_tok { include blacklisted-tokens.map; }
+#     map  $cookie_session_id $blck_lst_ses { include blacklisted-sessions.map; }
+#     ...
+#     # special log-format to notify fail2ban about failures:
+#     log_format f2b_session_errors '$msec failure "$cookie_session_id" - $remote_addr - $remote_user '
+#      ;#                  '"$request" $status $bytes_sent '
+#       #                  '"$http_referer" "$http_user_agent"';
+#
+#     # location checking blacklisted values:
+#     location ... {
+#       # check banned sessionid:
+#       if ($blck_lst_ses != "") {
+#         try_files "" @f2b-banned;
+#       }
+#       ...
+#       # notify fail2ban about a failure inside nginx:
+#       error_page 401 = @notify-f2b;
+#       ...
+#     }
+#     ...
+#     # location for return with "403 Forbidden" if banned:
+#     location @f2b-banned {
+#       default_type text/html;
+#       return 403 "<br/><center>
+#         <b style=\"color:red; font-size:18pt; border:2pt solid black; padding:5pt;\">
+#         You are banned!</b></center>";
+#     }
+#     ...
+#     # location to notify fail2ban about a failure inside nginx:
+#     location @notify-f2b {
+#       access_log /var/log/nginx/f2b-auth-errors.log f2b_session_errors;
+#     }
+#   }
+#   ...
+#
+# Note that quote-character (and possibly other special characters) are not allowed currently as session-id.
+# Thus please add any session-id validation rule in your locations (or in the corresponding backend-service), 
+# like in example below:
+#
+#   location ... {
+#     if ($cookie_session_id !~ "^[\w\-]+$") {
+#       return  403 "Wrong session-id"
+#     }
+#     ...
+#   }
+#
+# The parameters for jail corresponding log-format (f2b_session_errors):
+#
+#   [nginx-blck-lst]
+#   filter =
+#   datepattern = ^Epoch
+#   failregex = ^ failure "<F-ID>[^"]+</F-ID>" - <ADDR>
+#   usedns = no
+#
+# The same log-file can be used for IP-related jail (additionally to session-related, to ban very bad IPs):
+#
+#   [nginx-blck-ip]
+#   maxretry = 100
+#   filter =
+#   datepattern = ^Epoch
+#   failregex = ^ failure "[^"]+" - <ADDR>
+#   usedns = no
+#
+
+[Definition]
+
+# path to configuration of nginx (used to target nginx-instance in multi-instance system,
+# and as path for the blacklisted map):
+srv_cfg_path = /etc/nginx/
+
+# cmd-line arguments to supply to test/reload nginx:
+#srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf
+srv_cmd = nginx
+
+# first test configuration is correct, hereafter send reload signal:
+blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then
+                    %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi;
+                  fi;
+
+# map-file for nginx, can be redefined using `action = nginx-block-map[blck_lst_file="/path/file.map"]`:
+blck_lst_file = %(srv_cfg_path)s/blacklisted-sessions.map
+
+# Action definition:
+
+actionstart_on_demand = false
+actionstart = touch '%(blck_lst_file)s'
+
+actionflush = truncate -s 0 '%(blck_lst_file)s'; %(blck_lst_reload)s
+
+actionstop = %(actionflush)s
+
+actioncheck = 
+
+actionban = echo "\\\\<fid> 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s
+
+actionunban = id=$(echo "<fid>" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/$id 1;/d" %(blck_lst_file)s; %(blck_lst_reload)s
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 083ce815..befef926 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -115,11 +115,12 @@ fail2banserver.PRODUCTION = False
 
 def _out_file(fn, handle=logSys.debug):
 	"""Helper which outputs content of the file at HEAVYDEBUG loglevels"""
-	handle('---- ' + fn + ' ----')
-	for line in fileinput.input(fn):
-		line = line.rstrip('\n')
-		handle(line)
-	handle('-'*30)
+	if (handle != logSys.debug or logSys.getEffectiveLevel() <= logging.DEBUG):
+		handle('---- ' + fn + ' ----')
+		for line in fileinput.input(fn):
+			line = line.rstrip('\n')
+			handle(line)
+		handle('-'*30)
 
 
 def _write_file(fn, mode, *lines):
@@ -127,8 +128,19 @@ def _write_file(fn, mode, *lines):
 	f.write('\n'.join(lines))
 	f.close()
 
+def _read_file(fn):
+	f = None
+	try:
+		f = open(fn)
+		return f.read()
+	finally:
+		if f is not None:
+			f.close()
 
-def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
+
+def _start_params(tmp, use_stock=False, use_stock_cfg=None, 
+	logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None
+):
 	cfg = pjoin(tmp, "config")
 	if db == 'auto':
 		db = pjoin(tmp, "f2b-db.sqlite3")
@@ -138,8 +150,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 			"""Filters list of 'files' to contain only directories (under dir)"""
 			return [f for f in files if isdir(pjoin(dir, f))]
 		shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
-		os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "action.d")), pjoin(cfg, "action.d"))
-		os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "filter.d")), pjoin(cfg, "filter.d"))
+		use_stock_cfg = ('action.d', 'filter.d')
 		# replace fail2ban params (database with memory):
 		r = re.compile(r'^dbfile\s*=')
 		for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True):
@@ -170,13 +181,21 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 			"",
 		)
 		_write_file(pjoin(cfg, "jail.conf"), "w",
-			"[INCLUDES]", "",
-			"[DEFAULT]", "",
-			"",
+			*((
+				"[INCLUDES]", "",
+			  "[DEFAULT]", "tmp = " + tmp, "",
+			)+jails)
 		)
 		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(pjoin(cfg, "fail2ban.conf"))
 			_out_file(pjoin(cfg, "jail.conf"))
+	# link stock actions and filters:
+	if use_stock_cfg and STOCK:
+		for n in use_stock_cfg:
+			os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n))
+	if create_before_start:
+		for n in create_before_start:
+			_write_file(n % {'tmp': tmp}, 'w', '')
 	# parameters (sock/pid and config, increase verbosity, set log, etc.):
 	vvv, llev = (), "INFO"
 	if unittest.F2B.log_level < logging.INFO: # pragma: no cover
@@ -191,17 +210,13 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
 	)
 
 def _get_pid_from_file(pidfile):
-	f = pid = None
+	pid = None
 	try:
-		f = open(pidfile)
-		pid = f.read()
+		pid = _read_file(pidfile)
 		pid = re.match(r'\S+', pid).group()
 		return int(pid)
 	except Exception as e: # pragma: no cover
 		logSys.debug(e)
-	finally:
-		if f is not None:
-			f.close()
 	return pid
 
 def _kill_srv(pidfile):
@@ -297,12 +312,13 @@ def with_foreground_server_thread(startextra={}):
 				finally:
 					DefLogSys.info('=== within server: end.  ===')
 					self.pruneLog()
-					# stop:
-					self.execSuccess(startparams, "stop")
-					# wait for end:
-					Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
-					self.assertTrue(phase.get('end', None))
-					self.assertLogged("Shutdown successful", "Exiting Fail2ban")
+					# stop (if still running):
+					if not phase.get('end', None):
+						self.execSuccess(startparams, "stop")
+						# wait for end:
+						Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
+						self.assertTrue(phase.get('end', None))
+						self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True)
 			finally:
 				if th:
 					# we start client/server directly in current process (new thread),
@@ -1155,3 +1171,74 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Jail 'test-jail1' stopped", 
 			"Jail 'test-jail1' started", all=True)
+
+	# test action.d/nginx-block-map.conf --
+	@with_foreground_server_thread(startextra={
+		# create log-file (avoid "not found" errors):
+		'create_before_start': ('%(tmp)s/blck-failures.log',),
+		# we need action.d/nginx-block-map.conf:
+		'use_stock_cfg': ('action.d',),
+		# jail-config:
+		'jails': (
+			'[nginx-blck-lst]',
+			'backend = polling',
+      'usedns = no',
+			'logpath = %(tmp)s/blck-failures.log',
+			'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]',
+			'filter =',
+			'datepattern = ^Epoch',
+			'failregex = ^ failure "<F-ID>[^"]+</F-ID>" - <ADDR>',
+			'maxretry = 1', # ban by first failure
+			'enabled = true',
+	  )
+	})
+	def testServerActions_NginxBlockMap(self, tmp, startparams):
+		cfg = pjoin(tmp, "config")
+		lgfn = '%(tmp)s/blck-failures.log' % {'tmp': tmp}
+		mpfn = '%(tmp)s/blck-lst.map' % {'tmp': tmp}
+		# ban sessions (write log like nginx does it with f2b_session_errors log-format):
+		_write_file(lgfn, "w+",
+			str(int(MyTime.time())) + ' failure "125-000-001" - 192.0.2.1',
+			str(int(MyTime.time())) + ' failure "125-000-002" - 192.0.2.1',
+			str(int(MyTime.time())) + ' failure "125-000-003" - 192.0.2.1',
+			str(int(MyTime.time())) + ' failure "125-000-004" - 192.0.2.1',
+			str(int(MyTime.time())) + ' failure "125-000-005" - 192.0.2.1',
+		)
+		# check all sessions are banned (and blacklisted in map-file):
+		self.assertLogged(
+			"[nginx-blck-lst] Ban 125-000-001",
+			"[nginx-blck-lst] Ban 125-000-002",
+			"[nginx-blck-lst] Ban 125-000-003",
+			"[nginx-blck-lst] Ban 125-000-004",
+			"[nginx-blck-lst] Ban 125-000-005",
+			"Banned 5 / 5, 5 ticket(s)",
+			all=True, wait=MID_WAITTIME
+		)
+		_out_file(mpfn)
+		mp = _read_file(mpfn)
+		self.assertIn('\\125-000-001 1;\n', mp)
+		self.assertIn('\\125-000-002 1;\n', mp)
+		self.assertIn('\\125-000-003 1;\n', mp)
+		self.assertIn('\\125-000-004 1;\n', mp)
+		self.assertIn('\\125-000-005 1;\n', mp)
+
+		# unban 1, 2 and 5:
+		self.execSuccess(startparams, 'unban', '125-000-001', '125-000-002', '125-000-005')
+		_out_file(mpfn)
+		# check really unbanned but other sessions are still present (blacklisted in map-file):
+		mp = _read_file(mpfn)
+		self.assertNotIn('\\125-000-001 1;\n', mp)
+		self.assertNotIn('\\125-000-002 1;\n', mp)
+		self.assertNotIn('\\125-000-005 1;\n', mp)
+		self.assertIn('\\125-000-003 1;\n', mp)
+		self.assertIn('\\125-000-004 1;\n', mp)
+
+		# stop server and wait for end:
+		self.execSuccess(startparams, 'stop')
+		self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True, wait=MID_WAITTIME)
+
+		# check flushed (all sessions were deleted from map-file):
+		self.assertLogged("[nginx-blck-lst] Flush ticket(s) with nginx-block-map")
+		_out_file(mpfn)
+		mp = _read_file(mpfn)
+		self.assertEqual(mp, '')

From b62ab2d51e7fd27584dbbae3dae38139fc6861c1 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Nov 2017 13:46:57 +0100
Subject: [PATCH 557/623] ChangeLog updated

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9abafd79..4a4eee9c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -53,6 +53,9 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
 
 ### New Features
+* New Actions:
+  - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in
+    nginx-location with map-file);
 
 ### Enhancements
 * action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once.

From 55c2a9968a7a4afab43de393f3ce294fe92135ac Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Nov 2017 16:13:37 +0100
Subject: [PATCH 558/623] remove lacking [Init] section check ([Init] section
 not necessary anymore for actions also); fix sporadic error by shutdown
 server in with_foreground_server_thread decorator (if shutdown too fast, but
 end-phase still does not reached the tester-thread);

---
 fail2ban/tests/clientreadertestcase.py   | 2 --
 fail2ban/tests/fail2banclienttestcase.py | 3 +++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 4b2ec0e5..fdef3735 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -623,8 +623,6 @@ class JailsReaderTest(LogCaptureTestCase):
 						#print('****', actionName, opts.get('actionstart', ''))
 						self.assertIn('f2b-TEST', opts.get('actionstart', ''),
 							msg="Action file %r: interpolation of actionstart does not contains jail-name 'f2b-TEST'" % actionConfig)
-				self.assertIn('Init', actionReader.sections(),
-						msg="Action file %r is lacking [Init] section" % actionConfig)
 
 		def testReadStockJailConf(self):
 			jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index befef926..78a09e81 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -312,6 +312,9 @@ def with_foreground_server_thread(startextra={}):
 				finally:
 					DefLogSys.info('=== within server: end.  ===')
 					self.pruneLog()
+					# if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state):
+					if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")):
+						Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME)
 					# stop (if still running):
 					if not phase.get('end', None):
 						self.execSuccess(startparams, "stop")

From fbf89e8cdd1229e7207ecb3ec522761423d31e5f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 28 Nov 2017 16:32:16 +0100
Subject: [PATCH 559/623] typo in indent (spaces to tabs)

---
 fail2ban/tests/fail2banclienttestcase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 78a09e81..11b67fb2 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -1185,7 +1185,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		'jails': (
 			'[nginx-blck-lst]',
 			'backend = polling',
-      'usedns = no',
+			'usedns = no',
 			'logpath = %(tmp)s/blck-failures.log',
 			'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]',
 			'filter =',

From 660d57e6ba7bfa2dd6a16fdb6f5f9c27eecb335b Mon Sep 17 00:00:00 2001
From: SlowRiot <slowriot@voxelstorm.com>
Date: Wed, 29 Nov 2017 04:58:00 +0000
Subject: [PATCH 560/623] updating my email address

---
 config/filter.d/apache-shellshock.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/apache-shellshock.conf b/config/filter.d/apache-shellshock.conf
index 39df1704..725859a2 100644
--- a/config/filter.d/apache-shellshock.conf
+++ b/config/filter.d/apache-shellshock.conf
@@ -23,4 +23,4 @@ ignoreregex =
 # [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
 # [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'
 #
-# Author: Eugene Hopkinson (riot@riot.so)
+# Author: Eugene Hopkinson (e.hopkinson@gmail.com)

From aa158ac05fc8bfcd21c5ba8dac2f77934dc8fbcb Mon Sep 17 00:00:00 2001
From: Peter Nowee <peter+github@peternowee.com>
Date: Wed, 29 Nov 2017 14:07:00 +0100
Subject: [PATCH 561/623] Exim failregex: Include lower/mixed case AUTH

When reporting the error `AUTH command used when not advertised`, Exim
starts with `SMTP protocol error in "........."`. Here, Exim logs the
SMTP command as it was provided by the connecting client.
https://github.com/Exim/exim/blob/exim-4_89+fixes/src/src/smtp_in.c#L2850

According to RFC 5321 (SMTP) "[..] a command verb [..] MAY be encoded
in upper case, lower case, or any mixture of upper and lower case with
no impact on its meaning."
https://tools.ietf.org/html/rfc5321#section-2.4

Lower case `auth login` brute-force attempts were seen in the wild and
were not caught by the current failregex.

This commit makes the failregex case-insensitive for the `AUTH`
command, so that lower case (`auth`) or mixed case (`aUtH`) now also
match. The failregex was already case-insensitive for the command
arguments (e.g. `AUTH login` already matched).
---
 config/filter.d/exim.conf      | 2 +-
 fail2ban/tests/files/logs/exim | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index eb56feff..c195882a 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s SMTP protocol error in "[Aa][Uu][Tt][Hh] \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 
diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index f87d393b..e97fd7aa 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -76,3 +76,8 @@
 2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com)
 # failJSON: { "time": "2017-05-01T07:42:42", "match": true , "host": "192.0.2.3", "desc": "rejected RCPT - Unrouteable address" }
 2017-05-01 07:42:42 H=some.rev.dns.if.found (the.connector.reports.this.name) [192.0.2.3] F=<some.name@some.domain> rejected RCPT <some.invalid.name@a.domain>: Unrouteable address
+
+# failJSON: { "time": "2017-11-28T14:14:30", "match": true , "host": "192.0.2.4", "desc": "lower case AUTH command" }
+2017-11-28 14:14:30 SMTP protocol error in "auth login" H=(roxzgj) [192.0.2.4] AUTH command used when not advertised
+# failJSON: { "time": "2017-11-28T14:14:31", "match": true , "host": "192.0.2.5", "desc": "mixed case AUTH command" }
+2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised

From f59df2e156a140dce34a94fb146f4b4655e936b6 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 29 Nov 2017 20:55:48 +0100
Subject: [PATCH 562/623] Avoid any injecting on protocol (e. g. tries using
 camel-case)

The phrase "AUTH command used when not advertised" is precise enough as anchor here, so prevent by any foreign-input (any auth protocol error).
---
 config/filter.d/exim.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index c195882a..5e269f65 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "[Aa][Uu][Tt][Hh] \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s SMTP protocol error in "[^"]+" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 

From 4f631806110ee8bdb760b78da5e52632cff3aed8 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 30 Nov 2017 12:32:24 +0100
Subject: [PATCH 563/623] Avoid injection using quotes after `auth` command;

Added non-greedy fallback for quoted something (with lookahead simulated possessive greedy catch of non-quoted parts `[^"]*(?=")`).
Note that because host-info's are hereafter (with foreign input in-between), we would not use greedy or non-greedy catch-alls (`.*` or `.*?`) here (preventing performance losses).
---
 config/filter.d/exim.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 5e269f65..21fcaf73 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "[^"]+" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 

From cbd63d9cd545bf3e389d7f9510b83ad177410a8d Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 30 Nov 2017 12:45:11 +0100
Subject: [PATCH 564/623] added test to cover quoted injecting on AUTH command

---
 fail2ban/tests/files/logs/exim | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index e97fd7aa..97786f4e 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -81,3 +81,5 @@
 2017-11-28 14:14:30 SMTP protocol error in "auth login" H=(roxzgj) [192.0.2.4] AUTH command used when not advertised
 # failJSON: { "time": "2017-11-28T14:14:31", "match": true , "host": "192.0.2.5", "desc": "mixed case AUTH command" }
 2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised
+# failJSON: { "time": "2017-11-28T14:14:32", "match": true , "host": "192.0.2.6", "desc": "quoted injecting on AUTH command" }
+2017-11-28 14:14:32 SMTP protocol error in "aUtH lOgIn" H=(test) [8.8.8.8]" H=(roxzgj) [192.0.2.6] AUTH command used when not advertised

From 2e437937c36d7ca5a7d2a4740c2270d4e89d50ef Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 30 Nov 2017 17:06:37 +0100
Subject: [PATCH 565/623] datedetector: extended default date-patterns (allows
 extra space between the date and time stamps); * introduces 2 new format
 directives (with corresponding `%Ex` prefix for more precise parsing):   - %k
 - one- or two-digit number giving the hour of the day (0-23) on a 24-hour
 clock,    (corresponds %H, but allows space if not zero-padded).   - %l -
 one- or two-digit number giving the hour of the day (12-11) on a 12-hour
 clock,    (corresponds %I, but allows space if not zero-padded). *
 mysqld-auth test extended to cover new date-format in log. Closes gh-1639

---
 fail2ban/server/datedetector.py       | 24 ++++++++++++------------
 fail2ban/server/strptime.py           |  9 +++++++++
 fail2ban/tests/files/logs/mysqld-auth |  3 +++
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 90cfe1fd..278b2f20 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -124,22 +124,22 @@ class DateDetectorCache(object):
 		"""
 		self.__tmpcache = [], []
 		# ISO 8601, simple date, optional subsecond and timezone:
-		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59
+		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23  8:59:59
 		# simple date: 2005/01/23 21:59:59 
 		# custom for syslog-ng 2006.12.21 06:43:20
-		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T|  ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
-		self._cacheTemplate("(?:%a )?%b %d %ExY %H:%M:%S(?:\.%f)?")
+		self._cacheTemplate("(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?")
 		# simple date too (from x11vnc): 23/01/2005 21:59:59 
 		# and with optional year given by 2 digits: 23/01/05 21:59:59 
 		# (See http://bugs.debian.org/537610)
 		# 17-07-2008 17:23:25
-		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
+		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
 		# Apache format optional time zone:
 		# [31/Oct/2006:09:22:55 -0000]
 		# 26-Jul-2007 15:20:52
@@ -150,8 +150,8 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%m/%d/%ExY:%H:%M:%S")
 		# 01-27-2012 16:22:44.252
 		# subseconds explicit to avoid possible %m<->%d confusion
-		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
-		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
+		# with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
+		self._cacheTemplate("%m-%d-%ExY %k:%M:%S(?:\.%f)?")
 		# Epoch
 		self._cacheTemplate('EPOCH')
 		# Only time information in the log
@@ -163,14 +163,14 @@ class DateDetectorCache(object):
 		# Apache Tomcat
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
-		# 20050123T215959, 20050123 215959
-		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("^%b-%d-%Exy %k:%M:%S")
+		# 20050123T215959, 20050123 215959, , 20050123  85959
+		self._cacheTemplate("%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
-		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# +00:00 Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# TAI64N
 		self._cacheTemplate("TAI64N")
 		#
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index dbf75d21..498d284b 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -43,6 +43,13 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 
 timeRE = TimeRE()
 
+# %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
+# (corresponds %H, but allows space if not zero-padded).
+# %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
+# (corresponds %I, but allows space if not zero-padded).
+timeRE['k'] = r" ?(?P<H>[0-2]?\d)"
+timeRE['l'] = r" ?(?P<I>1?\d)"
+
 # TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
 #       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), 
 #       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' 
@@ -61,6 +68,8 @@ timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABB
 timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
 timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])"
 timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)"
+timeRE['Exk'] = r" ?(?P<H>2[0-3]|[0-1]\d|\d)"
+timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)"
 timeRE['ExM'] = r"(?P<M>[0-5]\d)"
 timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
 # more precise year patterns, within same century of last year and
diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth
index c77d214d..ebb8c0c4 100644
--- a/fail2ban/tests/files/logs/mysqld-auth
+++ b/fail2ban/tests/files/logs/mysqld-auth
@@ -17,6 +17,9 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for
 
 # failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" }
 2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES)
+# failJSON: { "time": "2015-10-07T08:25:38", "match": true , "host": "192.0.2.1", "desc": "log format with extra space between the date and time stamps, gh-1639" }
+2015-10-07  8:25:38 139845235329792 [Warning] Access denied for user 'user'@'192.0.2.1' (using password: YES)
+
 # failJSON: { "time": "2016-02-24T15:26:18", "match": true , "host": "localhost", "desc": "mysql 5.6 log format, Note instead of Warning" }
 2016-02-24T15:26:18.237955 6 [Note] Access denied for user 'root'@'localhost' (using password: YES)
 

From 55476974016103e92c23071fee9fa1098acbf318 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 1 Dec 2017 10:16:14 +0100
Subject: [PATCH 566/623] ChangeLog and typo

---
 ChangeLog                       | 7 +++++++
 fail2ban/server/datedetector.py | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4a4eee9c..d3ee6d02 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -41,6 +41,7 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 ### Fixes
 * Fixed logging to systemd-journal: new logtarget value SYSOUT can be used instead of STDOUT, to avoid 
   write of the time-stamp, if logging to systemd-journal from foreground mode (gh-1876)
+* Fixed recognition of the new date-format on mysqld-auth filter (gh-1639)
 * jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely 
   (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
 * config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf)
@@ -53,6 +54,12 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
 
 ### New Features
+* datedetector: extended default date-patterns (allows extra space between the date and time stamps);
+  introduces 2 new format directives (with corresponding %Ex prefix for more precise parsing):
+  - %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
+    (corresponds %H, but allows space if not zero-padded).
+  - %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
+    (corresponds %I, but allows space if not zero-padded).
 * New Actions:
   - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in
     nginx-location with map-file);
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 278b2f20..42df308e 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -164,7 +164,7 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
 		self._cacheTemplate("^%b-%d-%Exy %k:%M:%S")
-		# 20050123T215959, 20050123 215959, , 20050123  85959
+		# 20050123T215959, 20050123 215959, 20050123  85959
 		self._cacheTemplate("%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29

From e4bbaf3d58e78cb30b8a990129a905920d6b47b8 Mon Sep 17 00:00:00 2001
From: Peter Nowee <peter+github@peternowee.com>
Date: Fri, 1 Dec 2017 15:01:48 +0100
Subject: [PATCH 567/623] Update ChangeLog

---
 ChangeLog | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index f3868e4b..1010d6d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -33,7 +33,10 @@ releases.
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
     - extended to match pam_authenticate failures with "Permission denied" (gh-1897)
-* filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
+* filter.d/exim.conf
+    - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
+    - fixed failregex of "AUTH command used when not advertised" to better handle the foreign
+      input SMTP command (lower/mixed case auth command, prevent injection) (gh-1979)
 * filter.d/postfix-*.conf - added optional port regex (gh-1902)
 
 ### New Features

From 2b688825025c35072d6cdcd09e521e61d297d8a4 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 16:07:53 +0100
Subject: [PATCH 568/623] filter.d/exim.conf: provides mode "aggressive" to ban
 flood resp. DDOS-similar failures; Closes #1983

---
 config/filter.d/exim.conf      | 15 +++++++++++++++
 fail2ban/tests/files/logs/exim | 14 ++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf
index 19ade1e8..f1e56a75 100644
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@@ -24,6 +24,21 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
+            <mdre-<mode>>
+
+mdre-aggressive = ^%(pid)s no host name found for IP address <HOST>$
+                  ^%(pid)s no IP address found for host \S+ \(during SMTP connection from \[<HOST>\]\)$
+
+mdre-normal = 
+
+# Parameter `mode` - `normal` or `aggressive`.
+# Aggressive mode can be used to match flood and ddos-similar log-entries like:
+#   'no host found for IP', 'no IP found for host'.
+# Note this is not an authentication failures, so it may produce lots of false 
+# positives on misconfigured MTAs.
+# Ex.:
+#   filter = exim[mode=aggressive]
+mode = normal
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim
index 97786f4e..ea3c5dca 100644
--- a/fail2ban/tests/files/logs/exim
+++ b/fail2ban/tests/files/logs/exim
@@ -83,3 +83,17 @@
 2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised
 # failJSON: { "time": "2017-11-28T14:14:32", "match": true , "host": "192.0.2.6", "desc": "quoted injecting on AUTH command" }
 2017-11-28 14:14:32 SMTP protocol error in "aUtH lOgIn" H=(test) [8.8.8.8]" H=(roxzgj) [192.0.2.6] AUTH command used when not advertised
+
+## no matches with `mode = normal`:
+
+# failJSON: { "match": false , "desc": "aggressive mode only" }
+2017-12-03 08:32:00 no host name found for IP address 192.0.2.8
+# failJSON: { "match": false , "desc": "aggressive mode only" }
+2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
+
+# filterOptions: [{"mode": "aggressive"}]
+
+# failJSON: { "time": "2017-12-03T08:32:00", "match": true , "host": "192.0.2.8", "desc": "no host found for IP" }
+2017-12-03 08:32:00 no host name found for IP address 192.0.2.8
+# failJSON: { "time": "2017-12-03T08:51:35", "match": true , "host": "192.0.2.9", "desc": "no IP found for host" }
+2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])

From ffd6b9f6de3b94a255d145017ed6efb3a5c9e79c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 16:09:18 +0100
Subject: [PATCH 569/623] jail.conf: extended with new parameter `mode` for the
 filters supporting it;

---
 config/jail.conf | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/config/jail.conf b/config/jail.conf
index 1dabafe6..fc5dcb39 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -117,10 +117,13 @@ logencoding = auto
 enabled = false
 
 
+# "mode" defines the mode of the filter (see corresponding filter implementation for more info).
+mode = normal
+
 # "filter" defines the filter to use by the jail.
 #  By default jails have names matching their filter name
 #
-filter = %(__name__)s
+filter = %(__name__)s[mode=%(mode)s]
 
 
 #
@@ -237,8 +240,7 @@ action = %(action_)s
 # To use more aggressive sshd modes set filter parameter "mode" in jail.local:
 # normal (default), ddos, extra or aggressive (combines all).
 # See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details.
-mode    = normal
-filter  = sshd[mode=%(mode)s]
+#mode   = normal
 port    = ssh
 logpath = %(sshd_log)s
 backend = %(sshd_backend)s
@@ -536,7 +538,6 @@ backend  = %(syslog_backend)s
 [postfix]
 # To use another modes set filter parameter "mode" in jail.local:
 mode    = more
-filter  = postfix[mode=%(mode)s]
 port    = smtp,465,submission
 logpath = %(postfix_log)s
 backend = %(postfix_backend)s
@@ -562,8 +563,7 @@ backend = %(syslog_backend)s
 # To use more aggressive modes set filter parameter "mode" in jail.local:
 # normal (default), extra or aggressive
 # See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details.
-mode    = normal
-filter  = sendmail-reject[mode=%(mode)s]
+#mode    = normal
 port     = smtp,465,submission
 logpath  = %(syslog_mail)s
 backend  = %(syslog_backend)s
@@ -599,7 +599,8 @@ logpath = %(solidpop3d_log)s
 
 
 [exim]
-
+# see filter.d/exim.conf for further modes supported from filter:
+#mode = normal
 port   = smtp,465,submission
 logpath = %(exim_main_log)s
 
@@ -869,17 +870,14 @@ logpath  = /var/log/haproxy.log
 
 [slapd]
 port    = ldap,ldaps
-filter  = slapd
 logpath = /var/log/slapd.log
 
 [domino-smtp]
 port    = smtp,ssmtp
-filter  = domino-smtp
 logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log
 
 [phpmyadmin-syslog]
 port    = http,https
-filter  = phpmyadmin-syslog
 logpath = %(syslog_authpriv)s
 backend = %(syslog_backend)s
 

From b0ba1aa84611e308cc505bc1af0d8f4cd61640a4 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 5 Dec 2017 16:24:04 +0100
Subject: [PATCH 570/623] Update ChangeLog

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3512030d..3c13cd0a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -60,11 +60,13 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
     (corresponds %H, but allows space if not zero-padded).
   - %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
     (corresponds %I, but allows space if not zero-padded).
+* `filter.d/exim.conf`: added mode `aggressive` to ban flood resp. DDOS-similar failures (gh-1983);
 * New Actions:
   - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in
     nginx-location with map-file);
 
 ### Enhancements
+* jail.conf: extended with new parameter `mode` for the filters supporting it (gh-1988);
 * action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once.
 
 

From de97dedba0202559193e7654e183974ab335bcf7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 17:49:22 +0100
Subject: [PATCH 571/623] move extractOptions from JailReader to helpers
 (common usage server- / client-side);

---
 fail2ban/client/fail2banregex.py       |  8 +++----
 fail2ban/client/jailreader.py          | 33 +++-----------------------
 fail2ban/helpers.py                    | 28 ++++++++++++++++++++++
 fail2ban/server/jail.py                |  5 ++--
 fail2ban/tests/clientreadertestcase.py | 30 +++++++++++------------
 fail2ban/tests/servertestcase.py       |  6 ++---
 6 files changed, 55 insertions(+), 55 deletions(-)

diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py
index 89225ffb..84218149 100644
--- a/fail2ban/client/fail2banregex.py
+++ b/fail2ban/client/fail2banregex.py
@@ -46,12 +46,12 @@ except ImportError:
 	FilterSystemd = None
 
 from ..version import version
-from .jailreader import JailReader
 from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import Regex, RegexException
 
-from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, PREFER_ENC
+from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \
+  extractOptions, PREFER_ENC
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
@@ -287,7 +287,7 @@ class Fail2banRegex(object):
 		fltFile = None
 		fltOpt = {}
 		if regextype == 'fail':
-			fltName, fltOpt = JailReader.extractOptions(value)
+			fltName, fltOpt = extractOptions(value)
 			if fltName is not None:
 				if "." in fltName[~5:]:
 					tryNames = (fltName,)
@@ -606,7 +606,7 @@ class Fail2banRegex(object):
 				return False
 			output( "Use         systemd journal" )
 			output( "Use         encoding : %s" % self._encoding )
-			backend, beArgs = JailReader.extractOptions(cmd_log)
+			backend, beArgs = extractOptions(cmd_log)
 			flt = FilterSystemd(None, **beArgs)
 			flt.setLogEncoding(self._encoding)
 			myjournal = flt.getJournalReader()
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index ffc28752..382d3bc8 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -33,8 +33,7 @@ from .configreader import ConfigReaderUnshared, ConfigReader
 from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..version import version
-from ..helpers import getLogger
-from ..helpers import splitwords
+from ..helpers import getLogger, extractOptions, splitwords
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -42,15 +41,6 @@ logSys = getLogger(__name__)
 
 class JailReader(ConfigReader):
 	
-	# regex, to extract list of options:
-	optionCRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
-	# regex, to iterate over single option in option list, syntax:
-	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
-	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
-	# `action = act[p1=...][p2=...]`
-	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
-	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
 		self.__name = name
@@ -134,7 +124,7 @@ class JailReader(ConfigReader):
 			# Read filter
 			flt = self.__opts["filter"]
 			if flt:
-				filterName, filterOpt = JailReader.extractOptions(flt)
+				filterName, filterOpt = extractOptions(flt)
 				if not filterName:
 					raise JailDefError("Invalid filter definition %r" % flt)
 				self.__filter = FilterReader(
@@ -164,7 +154,7 @@ class JailReader(ConfigReader):
 				try:
 					if not act:			  # skip empty actions
 						continue
-					actName, actOpt = JailReader.extractOptions(act)
+					actName, actOpt = extractOptions(act)
 					if not actName:
 						raise JailDefError("Invalid action definition %r" % act)
 					if actName.endswith(".py"):
@@ -268,22 +258,5 @@ class JailReader(ConfigReader):
 		stream.insert(0, ["add", self.__name, backend])
 		return stream
 	
-	@staticmethod
-	def extractOptions(option):
-		match = JailReader.optionCRE.match(option)
-		if not match:
-			# TODO proper error handling
-			return None, None
-		option_name, optstr = match.groups()
-		option_opts = dict()
-		if optstr:
-			for optmatch in JailReader.optionExtractRE.finditer(optstr):
-				opt = optmatch.group(1)
-				value = [
-					val for val in optmatch.group(2,3,4) if val is not None][0]
-				option_opts[opt.strip()] = value.strip()
-		return option_name, option_opts
-
-
 class JailDefError(Exception):
 	pass
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index 5b027b32..378924df 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -237,6 +237,34 @@ else:
 			return uni_decode(x, enc, 'replace')
 
 
+#
+# Following function used for parse options from parameter (e.g. `name[p1=0, p2="..."][p3='...']`).
+#
+
+# regex, to extract list of options:
+OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
+# regex, to iterate over single option in option list, syntax:
+# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
+# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
+# `action = act[p1=...][p2=...]`
+OPTION_EXTRACT_CRE = re.compile(
+	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
+
+def extractOptions(option):
+	match = OPTION_CRE.match(option)
+	if not match:
+		# TODO proper error handling
+		return None, None
+	option_name, optstr = match.groups()
+	option_opts = dict()
+	if optstr:
+		for optmatch in OPTION_EXTRACT_CRE.finditer(optstr):
+			opt = optmatch.group(1)
+			value = [
+				val for val in optmatch.group(2,3,4) if val is not None][0]
+			option_opts[opt.strip()] = value.strip()
+	return option_name, option_opts
+
 #
 # Following facilities used for safe recursive interpolation of
 # tags (<tag>) in tagged options.
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index 39fdd959..eb990c47 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -27,8 +27,7 @@ import logging
 import Queue
 
 from .actions import Actions
-from ..client.jailreader import JailReader
-from ..helpers import getLogger, MyTime
+from ..helpers import getLogger, extractOptions, MyTime
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -85,7 +84,7 @@ class Jail(object):
 		return "%s(%r)" % (self.__class__.__name__, self.name)
 
 	def _setBackend(self, backend):
-		backend, beArgs = JailReader.extractOptions(backend)
+		backend, beArgs = extractOptions(backend)
 		backend = backend.lower()		# to assure consistent matching
 
 		backends = self._BACKENDS
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index fdef3735..e8cd2912 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -30,7 +30,7 @@ import tempfile
 import unittest
 from ..client.configreader import ConfigReader, ConfigReaderUnshared, NoSectionError
 from ..client import configparserinc
-from ..client.jailreader import JailReader
+from ..client.jailreader import JailReader, extractOptions
 from ..client.filterreader import FilterReader
 from ..client.jailsreader import JailsReader
 from ..client.actionreader import ActionReader, CommandAction
@@ -260,25 +260,25 @@ class JailReaderTest(LogCaptureTestCase):
 		# Simple example
 		option = "mail-whois[name=SSH]"
 		expected = ('mail-whois', {'name': 'SSH'})
-		result = JailReader.extractOptions(option)
+		result = extractOptions(option)
 		self.assertEqual(expected, result)
 
-		self.assertEqual(('mail.who_is', {}), JailReader.extractOptions("mail.who_is"))
-		self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), JailReader.extractOptions("mail.who_is[a=cat,b=dog]"))
-		self.assertEqual(('mail--ho_is', {}), JailReader.extractOptions("mail--ho_is"))
+		self.assertEqual(('mail.who_is', {}), extractOptions("mail.who_is"))
+		self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]"))
+		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is"))
 
-		self.assertEqual(('mail--ho_is', {}), JailReader.extractOptions("mail--ho_is['s']"))
+		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']"))
 		#self.printLog()
 		#self.assertLogged("Invalid argument ['s'] in ''s''")
 
-		self.assertEqual(('mail', {'a': ','}), JailReader.extractOptions("mail[a=',']"))
+		self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']"))
 
-		#self.assertRaises(ValueError, JailReader.extractOptions ,'mail-how[')
+		#self.assertRaises(ValueError, extractOptions ,'mail-how[')
 
 		# Empty option
 		option = "abc[]"
 		expected = ('abc', {})
-		result = JailReader.extractOptions(option)
+		result = extractOptions(option)
 		self.assertEqual(expected, result)
 
 		# More complex examples
@@ -296,11 +296,11 @@ class JailReaderTest(LogCaptureTestCase):
 			'opt10': "",
 			'opt11': "",
 		})
-		result = JailReader.extractOptions(option)
+		result = extractOptions(option)
 		self.assertEqual(expected, result)
 
 		# And multiple groups (`][` instead of `,`)
-		result = JailReader.extractOptions(option.replace(',', ']['))
+		result = extractOptions(option.replace(',', ']['))
 		expected2 = (expected[0],
 		 dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems())
 		)
@@ -439,7 +439,7 @@ class FilterReaderTest(unittest.TestCase):
 
 	def testFilterReaderSubstitionKnown(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>']]
-		filterName, filterOpt = JailReader.extractOptions(
+		filterName, filterOpt = extractOptions(
 			'substition[honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
 		filterReader = FilterReader('substition', "jailname", filterOpt,
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
@@ -650,7 +650,7 @@ class JailsReaderTest(LogCaptureTestCase):
 				if jail == 'INCLUDES':
 					continue
 				filterName = jails.get(jail, 'filter')
-				filterName, filterOpt = JailReader.extractOptions(filterName)
+				filterName, filterOpt = extractOptions(filterName)
 				allFilters.add(filterName)
 				self.assertTrue(len(filterName))
 				# moreover we must have a file for it
@@ -669,7 +669,7 @@ class JailsReaderTest(LogCaptureTestCase):
 				# somewhat duplicating here what is done in JailsReader if
 				# the jail is enabled
 				for act in actions.split('\n'):
-					actName, actOpt = JailReader.extractOptions(act)
+					actName, actOpt = extractOptions(act)
 					self.assertTrue(len(actName))
 					self.assertTrue(isinstance(actOpt, dict))
 					if actName == 'iptables-multiport':
@@ -696,7 +696,7 @@ class JailsReaderTest(LogCaptureTestCase):
 					if not (a.endswith('common.conf') or a.endswith('-aggressive.conf')))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
-				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails
+				extractOptions(jail.options['filter'])[0] for jail in jails.jails
 			)
 			self.maxDiff = None
 			self.assertTrue(filters.issubset(filters_jail),
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 2c293740..2d034806 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -42,7 +42,7 @@ from ..server.ticket import BanTicket
 from ..server.utils import Utils
 from .dummyjail import DummyJail
 from .utils import LogCaptureTestCase
-from ..helpers import getLogger, PREFER_ENC
+from ..helpers import getLogger, extractOptions, PREFER_ENC
 from .. import version
 
 try:
@@ -1034,7 +1034,7 @@ class LoggingTests(LogCaptureTestCase):
 					os.remove(f)
 
 
-from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK
+from clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR, STOCK
 
 class ServerConfigReaderTests(LogCaptureTestCase):
 
@@ -1145,7 +1145,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 
 		def getDefaultJailStream(self, jail, act):
 			act = act.replace('%(__name__)s', jail)
-			actName, actOpt = JailReader.extractOptions(act)
+			actName, actOpt = extractOptions(act)
 			stream = [
 				['add', jail, 'polling'],
 				# ['set', jail, 'addfailregex', 'DUMMY-REGEX <HOST>'],

From 1bf66364462ff0d4c224cce83315d1b29d41b06a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 18:54:21 +0100
Subject: [PATCH 572/623] Introduced new parameters for logging within
 fail2ban-server; Usage `logtarget = target[facility=..., datetime=on|off,
 format="..."]`:   - `facility` - specify syslog facility (default `daemon`,
 see https://docs.python.org/2/library/logging.handlers.html#sysloghandler    
  for the list of facilities);   - `datetime` - add date-time to the message
 (default on, ignored if `format` specified);   - `format` - specify own
 format how it will be logged, for example for short-log into STDOUT:      
 `fail2ban-server -f --logtarget 'stdout[format="%(relativeCreated)5d |
 %(message)s"]' start`; Closes gh-1980

---
 fail2ban/server/server.py                | 21 +++++++++++++++++----
 fail2ban/tests/fail2banclienttestcase.py |  5 +++--
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 9cb6f320..4a491903 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -37,7 +37,7 @@ from .filter import FileFilter, JournalFilter
 from .transmitter import Transmitter
 from .asyncserver import AsyncServer, AsyncServerException
 from .. import version
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, excepthook
+from ..helpers import getLogger, extractOptions, str2LogLevel, getVerbosityFormat, excepthook
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -547,6 +547,7 @@ class Server:
 	
 	def setLogTarget(self, target):
 		# check reserved targets in uppercase, don't change target, because it can be file:
+		target, logOptions = extractOptions(target)
 		systarget = target.upper()
 		with self.__loggingLock:
 			# don't set new handlers if already the same
@@ -559,7 +560,12 @@ class Server:
 			# set a format which is simpler for console use
 			fmt = "%(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
 			if systarget == "SYSLOG":
-				facility = logging.handlers.SysLogHandler.LOG_DAEMON
+				facility = logOptions.get('facility', 'DAEMON').upper()
+				try:
+					facility = getattr(logging.handlers.SysLogHandler, 'LOG_' + facility)
+				except AttributeError: # pragma: no cover
+					logSys.error("Unable to set facility %r, using 'DAEMON'", logOptions.get('facility'))
+					facility = logging.handlers.SysLogHandler.LOG_DAEMON
 				if self.__syslogSocket == "auto":
 					import platform
 					self.__syslogSocket = self.__autoSyslogSocketPaths.get(
@@ -610,9 +616,16 @@ class Server:
 				if self.__verbose is None:
 					self.__verbose = logging.DEBUG - logger.getEffectiveLevel() + 1
 			# If handler don't already add date to the message:
-			addtime = systarget not in ("SYSLOG", "SYSOUT")
+			addtime = logOptions.get('datetime')
+			if addtime is not None:
+				addtime = addtime in ('1', 'on', 'true', 'yes')
+			else:
+				addtime = systarget not in ("SYSLOG", "SYSOUT")
+			# If log-format is redefined in options:
+			if logOptions.get('format', '') != '':
+				fmt = logOptions.get('format')
 			# verbose log-format:
-			if self.__verbose is not None and self.__verbose > 2: # pragma: no cover
+			elif self.__verbose is not None and self.__verbose > 2: # pragma: no cover
 				fmt = getVerbosityFormat(self.__verbose-1,
 					addtime=addtime)
 			elif addtime:
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 11b67fb2..76e35856 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -171,7 +171,7 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None,
 		_write_file(pjoin(cfg, "fail2ban.conf"), "w",
 			"[Definition]",
 			"loglevel = INFO",
-			"logtarget = " + logtarget,
+			"logtarget = " + logtarget.replace('%', '%%'),
 			"syslogsocket = auto",
 			"socket = " + pjoin(tmp, "f2b.sock"),
 			"pidfile = " + pjoin(tmp, "f2b.pid"),
@@ -735,7 +735,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 	def testKillAfterStart(self, tmp):
 		try:
 			# to prevent fork of test-cases process, start server in background via command:
-			startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
+			startparams = _start_params(tmp, logtarget=pjoin(tmp,
+				'f2b.log[format="SRV: %(relativeCreated)3d | %(message)s", datetime=off]'))
 			# start (in new process, using the same python version):
 			cmd = (sys.executable, pjoin(BIN, SERVER))
 			logSys.debug('Start %s ...', cmd)

From f9833ddee4340b552026d648251aabea4a7d8b7c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 18:55:47 +0100
Subject: [PATCH 573/623] Update ChangeLog

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3c13cd0a..077c8ccd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -68,6 +68,13 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 ### Enhancements
 * jail.conf: extended with new parameter `mode` for the filters supporting it (gh-1988);
 * action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once.
+* Introduced new parameters for logging within fail2ban-server (gh-1980).
+  Usage `logtarget = target[facility=..., datetime=on|off, format="..."]`:
+  - `facility` - specify syslog facility (default `daemon`, see https://docs.python.org/2/library/logging.handlers.html#sysloghandler
+     for the list of facilities);
+  - `datetime` - add date-time to the message (default on, ignored if `format` specified);
+  - `format` - specify own format how it will be logged, for example for short-log into STDOUT:
+      `fail2ban-server -f --logtarget 'stdout[format="%(relativeCreated)5d | %(message)s"]' start`;
 
 
 ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th

From 55143ce1d93a5fe43e8eeb2bba2281e4a76bf0ef Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 5 Dec 2017 19:32:13 +0100
Subject: [PATCH 574/623] coverage increase

---
 fail2ban/tests/servertestcase.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 2d034806..9d7d9a76 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -831,8 +831,8 @@ class TransmitterLogging(TransmitterBase):
 		for logTarget in logTargets:
 			os.remove(logTarget)
 
-		self.setGetTest("logtarget", "STDOUT")
-		self.setGetTest("logtarget", "STDERR")
+		self.setGetTest("logtarget", 'STDOUT[format="%(message)s"]', 'STDOUT')
+		self.setGetTest("logtarget", 'STDERR[datetime=off]', 'STDERR')
 
 	def testLogTargetSYSLOG(self):
 		if not os.path.exists("/dev/log"):

From 6c705d572b7fd37d18396b9603b2fe79e3a78ff2 Mon Sep 17 00:00:00 2001
From: Kevin Maradona <RaidForums@users.noreply.github.com>
Date: Sun, 26 Nov 2017 22:29:52 +0100
Subject: [PATCH 575/623] filter.d/nginx-limit-req.conf: nginx limit-req
 log-level can be set to warn or error therefore having this regex will
 include both of them.

---
 config/filter.d/nginx-limit-req.conf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf
index 589d3d78..37962d10 100644
--- a/config/filter.d/nginx-limit-req.conf
+++ b/config/filter.d/nginx-limit-req.conf
@@ -36,10 +36,10 @@ ngx_limit_req_zones = [^"]+
 # Use following full expression if you should range limit request to specified 
 # servers, requests, referrers etc. only :
 #
-# failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+# failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
 
 # Shortly, much faster and stable version of regexp:
-failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>
+failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,
 
 ignoreregex = 
 

From e384acca5f085998174346d82102a370f7a47c05 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Tue, 5 Dec 2017 23:34:03 +0100
Subject: [PATCH 576/623] action.d/firewallcmd-ipset.conf: fixed create of set
 for ipv6 (missing `family inet6`)

---
 config/action.d/firewallcmd-ipset.conf | 4 +++-
 fail2ban/tests/servertestcase.py       | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 69447627..835df5a2 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -18,7 +18,7 @@ before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = ipset create <ipmset> hash:ip timeout <bantime>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
 actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
@@ -45,10 +45,12 @@ chain = INPUT_direct
 bantime = 600
 
 ipmset = f2b-<name>
+familyopt =
 
 [Init?family=inet6]
 
 ipmset = f2b-<name>6
+familyopt = <sp>family inet6
 
 
 # DEV NOTES:
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 2c293740..5cb69f00 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1653,7 +1653,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 					), 
 					'ip6-start': (
-						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600`",
+						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600 family inet6`",
 						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
 					'stop': (

From cc9ff31c9c62328cf7986ad90be168126485b3dc Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Tue, 5 Dec 2017 23:35:34 +0100
Subject: [PATCH 577/623] Update ChangeLog: `action.d/firewallcmd-ipset.conf`:
 fixed create of set for ipv6 (missing `family inet6`, gh-1990)

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 3c13cd0a..dbcca4d4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -49,6 +49,7 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 * `action.d/pf.conf`: 
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);
+* `action.d/firewallcmd-ipset.conf`: fixed create of set for ipv6 (missing `family inet6`, gh-1990)
 * `filter.d/sshd.conf`: extended failregex for modes "extra"/"aggressive": now finds all possible (also future)
   forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", 
   see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);

From aa9cefc3f842e331e0656dd7f96f9efd1f72748f Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Wed, 6 Dec 2017 01:09:04 +0100
Subject: [PATCH 578/623] proper stop server in the test cases (quit should
 stop all server-side threads, also if server was not really started); fix-up
 for run_with_except_hook: avoid very sporadic error "'NoneType' object has no
 attribute 'exc_info'" (https://bugs.python.org/issue7336), only extremely
 fast systems are affected ATM (2.x / 3.x), if thread ends nothing is
 available in .

---
 fail2ban/server/jailthread.py |  9 +++++++--
 fail2ban/server/server.py     | 30 ++++++++++++++++++------------
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py
index 280b2171..5c7afd38 100644
--- a/fail2ban/server/jailthread.py
+++ b/fail2ban/server/jailthread.py
@@ -67,8 +67,13 @@ class JailThread(Thread):
 		def run_with_except_hook(*args, **kwargs):
 			try:
 				run(*args, **kwargs)
-			except:
-				excepthook(*sys.exc_info())
+			except Exception as e:
+				# avoid very sporadic error "'NoneType' object has no attribute 'exc_info'" (https://bugs.python.org/issue7336)
+				# only extremely fast systems are affected ATM (2.7 / 3.x), if thread ends nothing is available here.
+				if sys is not None:
+					excepthook(*sys.exc_info())
+				else:
+					print(e)
 		self.run = run_with_except_hook
 
 	@abstractmethod
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 4a491903..de75bed6 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -82,7 +82,7 @@ class Server:
 
 	def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested
 		logSys.debug("Caught signal %d. Exiting", signum)
-		self.quit()
+		self.quit_signal()
 	
 	def __sigUSR1handler(self, signum, fname): # pragma: no cover - indirect tested
 		logSys.debug("Caught signal %d. Flushing logs", signum)
@@ -152,6 +152,20 @@ class Server:
 		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)
 
+		# Stop server
+		self.quit()
+
+		# Removes the PID file.
+		try:
+			logSys.debug("Remove PID file %s", pidfile)
+			os.remove(pidfile)
+		except (OSError, IOError) as e: # pragma: no cover
+			logSys.error("Unable to remove PID file: %s", e)
+		logSys.info("Exiting Fail2ban")
+
+	def quit(self):
+		self.quit_signal()
+
 		logSys.info("Shutdown in progress...")
 
 		# Restore default signal handlers:
@@ -168,17 +182,9 @@ class Server:
 			self.__db.close()
 			self.__db = None
 
-		# Removes the PID file.
-		try:
-			logSys.debug("Remove PID file %s", pidfile)
-			os.remove(pidfile)
-		except (OSError, IOError) as e: # pragma: no cover
-			logSys.error("Unable to remove PID file: %s", e)
-		logSys.info("Exiting Fail2ban")
-
-	def quit(self):
-		# Prevent to call quit twice:
-		self.quit = lambda: False
+	def quit_signal(self):
+		# Prevent to call quit_signal twice:
+		self.quit_signal = lambda: False
 		# Stop communication first because if jail's unban action
 		# tries to communicate via fail2ban-client we get a lockup
 		# among threads.  So the simplest resolution is to stop all

From 6ccaa03e00999c1309d50ccfc79b4d7e4247b7de Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Wed, 6 Dec 2017 01:10:56 +0100
Subject: [PATCH 579/623] action.d/firewallcmd-ipset.conf: extended with
 actionflush to bulk unban resp. flush ipset

---
 config/action.d/firewallcmd-ipset.conf | 4 +++-
 fail2ban/tests/servertestcase.py       | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 835df5a2..acb01466 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -21,8 +21,10 @@ before = firewallcmd-common.conf
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               firewall-cmd --direct --add-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
+actionflush = ipset flush <ipmset>
+
 actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
-             ipset flush <ipmset>
+             <actionflush>
              ipset destroy <ipmset>
 
 actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 65154f21..39601db9 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1656,6 +1656,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600 family inet6`",
 						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
+					'flush': (
+						"`ipset flush f2b-j-w-fwcmd-ipset`",
+						"`ipset flush f2b-j-w-fwcmd-ipset6`",
+					),
 					'stop': (
 						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-fwcmd-ipset`",

From bf6667d4da83a2c012d5fa245ca9002677aacec6 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Wed, 6 Dec 2017 01:38:39 +0100
Subject: [PATCH 580/623] better (sane) stop server handling,
 AsyncServer.stop_communication back-ported to 0.10 (cherry-picked from 0.11);

---
 fail2ban/server/asyncserver.py | 58 +++++++++++++++++++++++-----------
 fail2ban/server/server.py      | 30 ++++++++++--------
 2 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index 9cc74658..11c81649 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -37,7 +37,7 @@ import traceback
 
 from .utils import Utils
 from ..protocol import CSPROTO
-from ..helpers import getLogger,formatExceptionInfo
+from ..helpers import logging, getLogger, formatExceptionInfo
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -80,22 +80,36 @@ class RequestHandler(asynchat.async_chat):
 			# Deserialize
 			message = loads(message)
 			# Gives the message to the transmitter.
-			message = self.__transmitter.proceed(message)
+			if self.__transmitter:
+				message = self.__transmitter.proceed(message)
+			else:
+				message = ['SHUTDOWN']
 			# Serializes the response.
 			message = dumps(message, HIGHEST_PROTOCOL)
 			# Sends the response to the client.
 			self.push(message + CSPROTO.END)
-		except Exception as e: # pragma: no cover
+		except Exception as e:
 			logSys.error("Caught unhandled exception: %r", e,
 				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			# Sends the response to the client.
+			message = dumps("ERROR: %s" % e, HIGHEST_PROTOCOL)
+			self.push(message + CSPROTO.END)
 
-		
+	##
+	# Handles an communication errors in request.
+	#
 	def handle_error(self):
-		e1, e2 = formatExceptionInfo()
-		logSys.error("Unexpected communication error: %s" % str(e2))
-		logSys.error(traceback.format_exc().splitlines())
-		self.close()
-		
+		try:
+			e1, e2 = formatExceptionInfo()
+			logSys.error("Unexpected communication error: %s" % str(e2))
+			logSys.error(traceback.format_exc().splitlines())
+			# Sends the response to the client.
+			message = dumps("ERROR: %s" % e2, HIGHEST_PROTOCOL)
+			self.push(message + CSPROTO.END)
+		except Exception as e: # pragma: no cover - normally unreachable
+			pass
+		self.close_when_done()
+
 
 def loop(active, timeout=None, use_poll=False):
 	"""Custom event loop implementation
@@ -119,18 +133,20 @@ def loop(active, timeout=None, use_poll=False):
 			poll(timeout)
 			if errCount:
 				errCount -= 1
-		except Exception as e: # pragma: no cover
+		except Exception as e:
 			if not active():
 				break
 			errCount += 1
 			if errCount < 20:
-				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor')
+				# errno.ENOTCONN - 'Socket is not connected'
+				# errno.EBADF - 'Bad file descriptor'
+				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # pragma: no cover (too sporadic)
 					logSys.info('Server connection was closed: %s', str(e))
 				else:
 					logSys.error('Server connection was closed: %s', str(e))
 			elif errCount == 20:
-				logSys.info('Too many errors - stop logging connection errors')
 				logSys.exception(e)
+				logSys.error('Too many errors - stop logging connection errors')
 
 
 ##
@@ -158,10 +174,10 @@ class AsyncServer(asyncore.dispatcher):
 	def handle_accept(self):
 		try:
 			conn, addr = self.accept()
-		except socket.error:
+		except socket.error: # pragma: no cover
 			logSys.warning("Socket error")
 			return
-		except TypeError:
+		except TypeError: # pragma: no cover
 			logSys.warning("Type error")
 			return
 		AsyncServer.__markCloseOnExec(conn)
@@ -175,7 +191,7 @@ class AsyncServer(asyncore.dispatcher):
 	# @param sock: socket file.
 	# @param force: remove the socket file if exists.
 	
-	def start(self, sock, force, use_poll=False):
+	def start(self, sock, force, timeout=None, use_poll=False):
 		self.__worker = threading.current_thread()
 		self.__sock = sock
 		# Remove socket
@@ -191,7 +207,7 @@ class AsyncServer(asyncore.dispatcher):
 		self.set_reuse_addr()
 		try:
 			self.bind(sock)
-		except Exception:
+		except Exception: # pragma: no cover
 			raise AsyncServerException("Unable to bind socket %s" % self.__sock)
 		AsyncServer.__markCloseOnExec(self.socket)
 		self.listen(1)
@@ -201,12 +217,11 @@ class AsyncServer(asyncore.dispatcher):
 		if self.onstart:
 			self.onstart()
 		# Event loop as long as active:
-		loop(lambda: self.__loop, use_poll=use_poll)
+		loop(lambda: self.__loop, timeout=timeout, use_poll=use_poll)
 		self.__active = False
 		# Cleanup all
 		self.stop()
 
-
 	def close(self):
 		stopflg = False
 		if self.__active:
@@ -228,6 +243,13 @@ class AsyncServer(asyncore.dispatcher):
 	##
 	# Stops the communication server.
 	
+	def stop_communication(self):
+		logSys.debug("Stop communication")
+		self.__transmitter = None
+
+	##
+	# Stops the server.
+	
 	def stop(self):
 		self.close()
 
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index de75bed6..b166b94a 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -82,7 +82,7 @@ class Server:
 
 	def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested
 		logSys.debug("Caught signal %d. Exiting", signum)
-		self.quit_signal()
+		self.quit()
 	
 	def __sigUSR1handler(self, signum, fname): # pragma: no cover - indirect tested
 		logSys.debug("Caught signal %d. Flushing logs", signum)
@@ -152,7 +152,7 @@ class Server:
 		except AsyncServerException as e:
 			logSys.error("Could not start server: %s", e)
 
-		# Stop server
+		# Stop (if not yet already executed):
 		self.quit()
 
 		# Removes the PID file.
@@ -161,13 +161,22 @@ class Server:
 			os.remove(pidfile)
 		except (OSError, IOError) as e: # pragma: no cover
 			logSys.error("Unable to remove PID file: %s", e)
-		logSys.info("Exiting Fail2ban")
 
 	def quit(self):
-		self.quit_signal()
+		# Prevent to call quit twice:
+		self.quit = lambda: False
 
 		logSys.info("Shutdown in progress...")
 
+		# Stop communication first because if jail's unban action
+		# tries to communicate via fail2ban-client we get a lockup
+		# among threads.  So the simplest resolution is to stop all
+		# communications first (which should be ok anyways since we
+		# are exiting)
+		# See https://github.com/fail2ban/fail2ban/issues/7
+		if self.__asyncServer is not None:
+			self.__asyncServer.stop_communication()
+
 		# Restore default signal handlers:
 		if _thread_name() == '_MainThread':
 			for s, sh in self.__prev_signals.iteritems():
@@ -182,18 +191,11 @@ class Server:
 			self.__db.close()
 			self.__db = None
 
-	def quit_signal(self):
-		# Prevent to call quit_signal twice:
-		self.quit_signal = lambda: False
-		# Stop communication first because if jail's unban action
-		# tries to communicate via fail2ban-client we get a lockup
-		# among threads.  So the simplest resolution is to stop all
-		# communications first (which should be ok anyways since we
-		# are exiting)
-		# See https://github.com/fail2ban/fail2ban/issues/7
+		# Stop async
 		if self.__asyncServer is not None:
 			self.__asyncServer.stop()
 			self.__asyncServer = None
+		logSys.info("Exiting Fail2ban")
 
 	def addJail(self, name, backend):
 		addflg = True
@@ -610,7 +612,7 @@ class Server:
 				try:
 					handler.flush()
 					handler.close()
-				except (ValueError, KeyError):  # pragma: no cover
+				except (ValueError, KeyError): # pragma: no cover
 					# Is known to be thrown after logging was shutdown once
 					# with older Pythons -- seems to be safe to ignore there
 					# At least it was still failing on 2.6.2-0ubuntu1 (jaunty)

From 7c63eb23784b5de4d65fd77347bae68f0164e75b Mon Sep 17 00:00:00 2001
From: root <root@localhost>
Date: Fri, 8 Dec 2017 16:55:07 +0900
Subject: [PATCH 581/623] In the CentOS7 and epel environment, result of
 "firewall-cmd -direct -get -chains ipv4 filter" is displayed one line

Changed to be multiple lines with reference to firewallcmd-multiport.conf
---
 config/action.d/firewallcmd-new.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf
index e64601e1..261573ba 100644
--- a/config/action.d/firewallcmd-new.conf
+++ b/config/action.d/firewallcmd-new.conf
@@ -16,7 +16,7 @@ actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m st
              firewall-cmd --direct --remove-rules <family> filter f2b-<name>
              firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
-actioncheck = firewall-cmd --direct --get-chains <family> filter | grep -q 'f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q 'f2b-<name>$'
 
 actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 

From 79f414c6a289763d709104fafbac30de4c0092e1 Mon Sep 17 00:00:00 2001
From: root <root@localhost>
Date: Fri, 8 Dec 2017 17:19:41 +0900
Subject: [PATCH 582/623] fix <family> typo

---
 config/action.d/firewallcmd-new.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf
index 261573ba..b06f5ccd 100644
--- a/config/action.d/firewallcmd-new.conf
+++ b/config/action.d/firewallcmd-new.conf
@@ -16,7 +16,7 @@ actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 -m st
              firewall-cmd --direct --remove-rules <family> filter f2b-<name>
              firewall-cmd --direct --remove-chain <family> filter f2b-<name>
 
-actioncheck = firewall-cmd --direct --get-chains ipv4 filter | sed -e 's, ,\n,g' | grep -q 'f2b-<name>$'
+actioncheck = firewall-cmd --direct --get-chains <family> filter | sed -e 's, ,\n,g' | grep -q 'f2b-<name>$'
 
 actionban = firewall-cmd --direct --add-rule <family> filter f2b-<name> 0 -s <ip> -j <blocktype>
 

From 9374de59f3299a8c52c681977ba9e73764b00a72 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 21 Dec 2017 22:38:54 +0100
Subject: [PATCH 583/623] Automatically recover or recreate corrupt persistent
 database (e. g. if failed to open with 'database disk image is malformed').
 Closes #1465

---
 fail2ban/server/database.py        | 102 +++++++++++++++++++++++------
 fail2ban/tests/databasetestcase.py |  73 +++++++++++++++++----
 2 files changed, 144 insertions(+), 31 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index f4f9b6c2..5474311b 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
 import json
+import os
 import shutil
 import sqlite3
 import sys
@@ -31,6 +32,7 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
+from .utils import Utils
 from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
@@ -163,13 +165,17 @@ class Fail2BanDb(object):
 
 	def __init__(self, filename, purgeAge=24*60*60):
 		self.maxEntries = 50
+		self._lock = RLock()
+		self._dbFilename = filename
+		self._purgeAge = purgeAge
+		self._connectDB()
+
+	def _connectDB(self, checkIntegrity=False):
+		filename = self._dbFilename
 		try:
-			self._lock = RLock()
 			self._db = sqlite3.connect(
 				filename, check_same_thread=False,
 				detect_types=sqlite3.PARSE_DECLTYPES)
-			self._dbFilename = filename
-			self._purgeAge = purgeAge
 
 			self._bansMergedCache = {}
 
@@ -190,20 +196,38 @@ class Fail2BanDb(object):
 			pypy = False
 
 		cur = self._db.cursor()
-		cur.execute("PRAGMA foreign_keys = ON")
-		# speedup: write data through OS without syncing (no wait):
-		cur.execute("PRAGMA synchronous = OFF")
-		# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
-		if not pypy:
-			cur.execute("PRAGMA journal_mode = MEMORY")
-		# speedup: temporary tables and indices are kept in memory:
-		cur.execute("PRAGMA temp_store = MEMORY")
-
 		try:
+			cur.execute("PRAGMA foreign_keys = ON")
+			# speedup: write data through OS without syncing (no wait):
+			cur.execute("PRAGMA synchronous = OFF")
+			# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
+			if not pypy:
+				cur.execute("PRAGMA journal_mode = MEMORY")
+			# speedup: temporary tables and indices are kept in memory:
+			cur.execute("PRAGMA temp_store = MEMORY")
+
+			if checkIntegrity:
+				logSys.debug("  Check integrity ...")
+				cur.execute("PRAGMA integrity_check")
+				for s in cur.fetchall():
+					logSys.debug("  %s", s)
+				self._db.commit()
+
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:
 			logSys.warning("New database created. Version '%i'",
 				self.createDb())
+		except sqlite3.Error as e:
+			logSys.error(
+				"Error opening fail2ban persistent database '%s': %s",
+				filename, e.args[0])
+			# if not a file - raise an error:
+			if not os.path.isfile(filename):
+				raise
+			# try to repair it:
+			cur.close()
+			cur = None
+			self.repairDB()
 		else:
 			version = cur.fetchone()[0]
 			if version < Fail2BanDb.__version__:
@@ -217,16 +241,55 @@ class Fail2BanDb(object):
 						Fail2BanDb.__version__, version, newversion)
 					raise RuntimeError('Failed to fully update')
 		finally:
-			# pypy: set journal mode after possible upgrade db:
-			if pypy:
-				cur.execute("PRAGMA journal_mode = MEMORY")
-			cur.close()
+			if cur:
+				# pypy: set journal mode after possible upgrade db:
+				if pypy:
+					cur.execute("PRAGMA journal_mode = MEMORY")
+				cur.close()
 
 	def close(self):
 		logSys.debug("Close connection to database ...")
 		self._db.close()
 		logSys.info("Connection to database closed.")
 
+	@property
+	def _dbBackupFilename(self):
+		try:
+			return self.__dbBackupFilename
+		except AttributeError:
+			self.__dbBackupFilename = self._dbFilename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+			return self.__dbBackupFilename
+	
+	def repairDB(self):
+		# avoid endless recursion if reconnect failed again for some reasons:
+		_repairDB = self.repairDB
+		self.repairDB = None
+		try:
+			# backup
+			logSys.info("Trying to repair database %s", self._dbFilename)
+			shutil.move(self._dbFilename, self._dbBackupFilename)
+			logSys.info("  Database backup created: %s", self._dbBackupFilename)
+
+			# first try to repair using dump/restore in order 
+			Utils.executeCmd((r"""f2b_db=$0; f2b_dbbk=$1; sqlite3 "$f2b_dbbk" ".dump" | sqlite3 "$f2b_db" """,
+				self._dbFilename, self._dbBackupFilename))
+			dbFileSize = os.stat(self._dbFilename).st_size
+			if dbFileSize:
+				logSys.info("  Repair seems to be successful, restored %d byte(s).", dbFileSize)
+				# succeeded - try to reconnect:
+				self._connectDB(checkIntegrity=True)
+			else:
+				logSys.info("  Repair seems to be failed, restored %d byte(s).", dbFileSize)
+				raise Exception('Recreate ...')
+		except Exception as e:
+			# if still failed, just recreate database as fallback:
+			logSys.error("  Error repairing of fail2ban database '%s': %s",
+				self._dbFilename, e.args[0])
+			os.remove(self._dbFilename)
+			self._connectDB()
+		finally:
+			self.repairDB = _repairDB
+
 	@property
 	def filename(self):
 		"""File name of SQLite3 database file.
@@ -271,9 +334,10 @@ class Fail2BanDb(object):
 			raise NotImplementedError(
 						"Attempt to travel to future version of database ...how did you get here??")
 
-		self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
-		shutil.copyfile(self.filename, self._dbBackupFilename)
-		logSys.info("Database backup created: %s", self._dbBackupFilename)
+		logSys.info("Uprade database: %s", self._dbBackupFilename)
+		if not os.path.isfile(self._dbBackupFilename):
+			shutil.copyfile(self.filename, self._dbBackupFilename)
+			logSys.info("  Database backup created: %s", self._dbBackupFilename)
 
 		if version < 2:
 			cur.executescript("BEGIN TRANSACTION;"
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 33fc4413..5ac590f5 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -62,7 +62,18 @@ class DatabaseTest(LogCaptureTestCase):
 		self.dbFilename = None
 		if not unittest.F2B.memory_db:
 			_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
-		self.db = getFail2BanDb(self.dbFilename)
+		self._db = ':auto-create-in-memory:'
+
+	@property
+	def db(self):
+		if isinstance(self._db, basestring) and self._db == ':auto-create-in-memory:':
+			self._db = getFail2BanDb(self.dbFilename)
+		return self._db
+	@db.setter
+	def db(self, value):
+		if isinstance(self._db, Fail2BanDb): # pragma: no cover
+			self._db.close()
+		self._db = value
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -106,23 +117,61 @@ class DatabaseTest(LogCaptureTestCase):
 			self.jail.name in self.db.getJailNames(),
 			"Jail not retained in Db after disconnect reconnect.")
 
-	def testUpdateDb(self):
+	def testRepairDb(self):
 		if Fail2BanDb is None: # pragma: no cover
 			return
 		self.db = None
 		if self.dbFilename is None: # pragma: no cover
 			_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
-		shutil.copyfile(
-			os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename)
-		self.db = Fail2BanDb(self.dbFilename)
-		self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets']))
-		self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log']))
-		ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"])
-		self.assertEqual(self.db.getBans()[0], ticket)
+		# test truncated database with different sizes:
+		#   - 14000 bytes - seems to be reparable,
+		#   - 4000  bytes - is totally broken.
+		for truncSize in (14000, 4000):
+			self.pruneLog("[test-repair], next phase - file-size: %d" % truncSize)
+			shutil.copyfile(
+				os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename)
+			# produce currupt database:
+			f = os.open(self.dbFilename, os.O_RDWR)
+			os.ftruncate(f, truncSize)
+			os.close(f)
+			# test repair:
+			try:
+				self.db = Fail2BanDb(self.dbFilename)
+				if truncSize == 14000: # restored:
+					self.assertLogged("Repair seems to be successful",
+						"Check integrity", "Database updated", all=True)
+					self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log']))
+					self.assertEqual(len(self.db.getJailNames()), 1)
+				else: # recreated:
+					self.assertLogged("Repair seems to be failed",
+						"New database created.", all=True)
+					self.assertEqual(len(self.db.getLogPaths()), 0)
+					self.assertEqual(len(self.db.getJailNames()), 0)
+			finally:
+				if self.db and self.db._dbFilename != ":memory:":
+					os.remove(self.db._dbBackupFilename)
+					self.db = None
 
-		self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__)
-		self.assertRaises(NotImplementedError, self.db.updateDb, Fail2BanDb.__version__ + 1)
-		os.remove(self.db._dbBackupFilename)
+	def testUpdateDb(self):
+		if Fail2BanDb is None: # pragma: no cover
+			return
+		self.db = None
+		try:
+			if self.dbFilename is None: # pragma: no cover
+				_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
+			shutil.copyfile(
+				os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename)
+			self.db = Fail2BanDb(self.dbFilename)
+			self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets']))
+			self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log']))
+			ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"])
+			self.assertEqual(self.db.getBans()[0], ticket)
+
+			self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__)
+			self.assertRaises(NotImplementedError, self.db.updateDb, Fail2BanDb.__version__ + 1)
+		finally:
+			if self.db and self.db._dbFilename != ":memory:":
+				os.remove(self.db._dbBackupFilename)
 
 	def testAddJail(self):
 		if Fail2BanDb is None: # pragma: no cover

From 79443210ada8e2ac43ebf140f7404dd5aac2c89a Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 21 Dec 2017 22:49:57 +0100
Subject: [PATCH 584/623] Update ChangeLog

---
 ChangeLog | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index eafc31d1..4d9dfbb3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -76,7 +76,9 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   - `datetime` - add date-time to the message (default on, ignored if `format` specified);
   - `format` - specify own format how it will be logged, for example for short-log into STDOUT:
       `fail2ban-server -f --logtarget 'stdout[format="%(relativeCreated)5d | %(message)s"]' start`;
-
+* Automatically recover or recreate corrupt persistent database (e. g. if failed to open with 
+  'database disk image is malformed'). Fail2ban will create a backup, try to repair the database,
+  if repair fails - recreate new database (gh-1465, gh-2004).
 
 ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
 -----------

From 2d23f35d2680e761ab94ef1e2ed17668dd8cb465 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 21 Dec 2017 22:50:54 +0100
Subject: [PATCH 585/623] Update ChangeLog

typo: missing newline restored.
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 4d9dfbb3..f4a8e218 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -80,6 +80,7 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   'database disk image is malformed'). Fail2ban will create a backup, try to repair the database,
   if repair fails - recreate new database (gh-1465, gh-2004).
 
+
 ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
 -----------
 

From 1ad587ac7c4f4ee4cc2cd3f8f18d41981e1a4d30 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 11:20:52 +0100
Subject: [PATCH 586/623] Stability fix for fail2banclienttestcase:   - provide
 waitForServerEnd method for decorator `with_foreground_server_thread`, to
 wait for real server stop if needed;   - accept any exit code in decorator
 `with_foreground_server_thread`, because multi-threaded, thus server can exit
 in-between;   - fix sporadic fail "AssertionError: 'Banned 5 / 5, 5
 ticket(s)' was not found" (if some tickets will be processed earlier,    
 thus not as chunk but separately), so in case of: Banned 1 / 1, 1 ticket(s)
 in 'nginx-blck-lst' Banned 4 / 5, 5 ticket(s) in 'nginx-blck-lst'

---
 fail2ban/tests/fail2banclienttestcase.py | 186 +++++++++++------------
 1 file changed, 90 insertions(+), 96 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 76e35856..11d1983e 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -96,6 +96,9 @@ class FailExitException(fail2bancmdline.ExitException):
 	pass
 
 
+SUCCESS = ExitException
+FAILED = FailExitException
+
 INTERACT = []
 
 
@@ -299,35 +302,40 @@ def with_foreground_server_thread(startextra={}):
 				)
 				th.daemon = True
 				th.start()
-				try:
-					# wait for start thread:
-					Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
-					self.assertTrue(phase.get('start', None))
-					# wait for server (socket and ready):
-					self._wait_for_srv(tmp, True, startparams=startparams)
-					DefLogSys.info('=== within server: begin ===')
-					self.pruneLog()
-					# several commands to server in body of decorated function:
-					return f(self, tmp, startparams, *args, **kwargs)
-				finally:
-					DefLogSys.info('=== within server: end.  ===')
-					self.pruneLog()
+				# to wait for end of server, default accept any exit code, because multi-threaded, 
+				# thus server can exit in-between...
+				def _waitForServerEnd(code=(SUCCESS, FAILED)):
 					# if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state):
 					if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")):
 						Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME)
 					# stop (if still running):
 					if not phase.get('end', None):
-						self.execSuccess(startparams, "stop")
-						# wait for end:
+						self.execCmd(code, startparams, "stop")
+						# wait for end sign:
 						Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
 						self.assertTrue(phase.get('end', None))
 						self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True)
+				self.waitForServerEnd = _waitForServerEnd
+				# wait for start thread:
+				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
+				self.assertTrue(phase.get('start', None))
+				# wait for server (socket and ready):
+				self._wait_for_srv(tmp, True, startparams=startparams)
+				DefLogSys.info('=== within server: begin ===')
+				self.pruneLog()
+				# several commands to server in body of decorated function:
+				return f(self, tmp, startparams, *args, **kwargs)
 			finally:
 				if th:
+					# wait for server end (if not yet already exited):
+					DefLogSys.info('=== within server: end.  ===')
+					self.pruneLog()
+					self.waitForServerEnd()
 					# we start client/server directly in current process (new thread),
 					# so don't kill (same process) - if success, just wait for end of worker:
 					if phase.get('end', None):
 						th.join()
+				self.waitForServerEnd = None
 		return wrapper
 	return _deco_wrapper
 
@@ -388,11 +396,9 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 				logSys.debug("No log file %s to examine details of error", log)
 			raise
 
-	def execSuccess(self, startparams, *args):
-		raise NotImplementedError("To be defined in subclass")
-
-	def execFailed(self, startparams, *args):
-		raise NotImplementedError("To be defined in subclass")
+	def execCmd(self, exitType, startparams, *args):
+		self.assertRaises(exitType, self.exec_command_line[0],
+			(self.exec_command_line[1:] + startparams + args))
 
 	#
 	# Common tests
@@ -401,7 +407,7 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 		# start and wait to end (foreground):
 		logSys.debug("start of test worker")
 		phase['start'] = True
-		self.execSuccess(("-f",) + startparams, "start")
+		self.execCmd(SUCCESS, ("-f",) + startparams, "start")
 		# end :
 		phase['end'] = True
 		logSys.debug("end of test worker")
@@ -409,46 +415,40 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 	@with_foreground_server_thread()
 	def testStartForeground(self, tmp, startparams):
 		# several commands to server:
-		self.execSuccess(startparams, "ping")
-		self.execFailed(startparams, "~~unknown~cmd~failed~~")
-		self.execSuccess(startparams, "echo", "TEST-ECHO")
+		self.execCmd(SUCCESS, startparams, "ping")
+		self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~")
+		self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO")
 
 
 class Fail2banClientTest(Fail2banClientServerBase):
 
-	def execSuccess(self, startparams, *args):
-		self.assertRaises(ExitException, _exec_client,
-			((CLIENT,) + startparams + args))
-
-	def execFailed(self, startparams, *args):
-		self.assertRaises(FailExitException, _exec_client,
-			((CLIENT,) + startparams + args))
+	exec_command_line = (_exec_client, CLIENT,)
 
 	def testConsistency(self):
 		self.assertTrue(isfile(pjoin(BIN, CLIENT)))
 		self.assertTrue(isfile(pjoin(BIN, SERVER)))
 
 	def testClientUsage(self):
-		self.execSuccess((), "-h")
+		self.execCmd(SUCCESS, (), "-h")
 		self.assertLogged("Usage: " + CLIENT)
 		self.assertLogged("Report bugs to ")
 		self.pruneLog()
-		self.execSuccess((), "-vq", "-V")
+		self.execCmd(SUCCESS, (), "-vq", "-V")
 		self.assertLogged("Fail2Ban v" + fail2bancmdline.version)
 		self.pruneLog()
-		self.execSuccess((), "--str2sec", "1d12h30m")
+		self.execCmd(SUCCESS, (), "--str2sec", "1d12h30m")
 		self.assertLogged("131400")
 
 	@with_tmpdir
 	def testClientDump(self, tmp):
 		# use here the stock configuration (if possible)
 		startparams = _start_params(tmp, True)
-		self.execSuccess(startparams, "-vvd")
+		self.execCmd(SUCCESS, startparams, "-vvd")
 		self.assertLogged("Loading files")
 		self.assertLogged("['set', 'logtarget',")
 		self.pruneLog()
 		# pretty dump:
-		self.execSuccess(startparams, "--dp")
+		self.execCmd(SUCCESS, startparams, "--dp")
 		self.assertLogged("['set', 'logtarget',")
 		
 	@with_tmpdir
@@ -457,28 +457,28 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		# use once the stock configuration (to test starting also)
 		startparams = _start_params(tmp, True)
 		# start:
-		self.execSuccess(("-b",) + startparams, "start")
+		self.execCmd(SUCCESS, ("-b",) + startparams, "start")
 		# wait for server (socket and ready):
 		self._wait_for_srv(tmp, True, startparams=startparams)
 		self.assertLogged("Server ready")
 		self.assertLogged("Exit with code 0")
 		try:
-			self.execSuccess(startparams, "echo", "TEST-ECHO")
-			self.execFailed(startparams, "~~unknown~cmd~failed~~")
+			self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO")
+			self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~")
 			self.pruneLog()
 			# start again (should fail):
-			self.execFailed(("-b",) + startparams, "start")
+			self.execCmd(FAILED, ("-b",) + startparams, "start")
 			self.assertLogged("Server already running")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.execSuccess(startparams, "stop")
+			self.execCmd(SUCCESS, startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
 		self.pruneLog()
 		# stop again (should fail):
-		self.execFailed(startparams, "stop")
+		self.execCmd(FAILED, startparams, "stop")
 		self.assertLogged("Failed to access socket path")
 		self.assertLogged("Is fail2ban running?")
 
@@ -489,7 +489,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget=pjoin(tmp, "f2b.log"))
 		# if fast, start server process from client started direct here:
 		if unittest.F2B.fast: # pragma: no cover
-			self.execSuccess(startparams + ("start",))
+			self.execCmd(SUCCESS, startparams + ("start",))
 		else:
 			# start (in new process, using the same python version):
 			cmd = (sys.executable, pjoin(BIN, CLIENT))
@@ -503,12 +503,12 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		self.pruneLog()
 		try:
 			# echo from client (inside):
-			self.execSuccess(startparams, "echo", "TEST-ECHO")
+			self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO")
 			self.assertLogged("TEST-ECHO")
 			self.assertLogged("Exit with code 0")
 			self.pruneLog()
 			# test ping timeout:
-			self.execSuccess(startparams, "ping", "0.1")
+			self.execCmd(SUCCESS, startparams, "ping", "0.1")
 			self.assertLogged("Server replied: pong")
 			self.pruneLog()
 			# python 3 seems to bypass such short timeouts also, 
@@ -519,7 +519,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				os.kill(pid, signal.SIGSTOP); # or SIGTSTP?
 				time.sleep(Utils.DEFAULT_SHORT_INTERVAL)
 				# test ping with short timeout:
-				self.execFailed(startparams, "ping", "1e-10")
+				self.execCmd(FAILED, startparams, "ping", "1e-10")
 			finally:
 				# resume:
 				os.kill(pid, signal.SIGCONT)
@@ -531,7 +531,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"status",
 				"exit"
 			]
-			self.execSuccess(startparams, "-i")
+			self.execCmd(SUCCESS, startparams, "-i")
 			self.assertLogged("INTERACT-ECHO")
 			self.assertLogged("Status", "Number of jail:")
 			self.assertLogged("Exit with code 0")
@@ -542,7 +542,7 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"restart",
 				"exit"
 			]
-			self.execSuccess(startparams, "-i")
+			self.execCmd(SUCCESS, startparams, "-i")
 			self.assertLogged("Reading config files:")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Server ready")
@@ -553,18 +553,18 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				"reload ~~unknown~jail~fail~~",
 				"exit"
 			]
-			self.execSuccess(startparams, "-i")
+			self.execCmd(SUCCESS, startparams, "-i")
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.pruneLog()
 			# test reload missing jail (direct):
-			self.execFailed(startparams, "reload", "~~unknown~jail~fail~~")
+			self.execCmd(FAILED, startparams, "reload", "~~unknown~jail~fail~~")
 			self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'")
 			self.assertLogged("Exit with code -1")
 			self.pruneLog()
 		finally:
 			self.pruneLog()
 			# stop:
-			self.execSuccess(startparams, "stop")
+			self.execCmd(SUCCESS, startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
@@ -575,33 +575,33 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"--async", "-c", pjoin(tmp, "miss"), "start")
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start")
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## not running
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload")
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "start")
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
 		os.remove(pjoin(tmp, "f2b.sock"))
 
 		## wrong option:
-		self.execFailed((), "-s")
+		self.execCmd(FAILED, (), "-s")
 		self.assertLogged("Usage: ")
 		self.pruneLog()
 
@@ -611,13 +611,13 @@ class Fail2banClientTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		# not started:
-		self.execFailed(startparams,
+		self.execCmd(FAILED, startparams,
 			"reload", "jail")
 		self.assertLogged("Could not find server")
 		self.pruneLog()
 
 		# unexpected arg:
-		self.execFailed(startparams,
+		self.execCmd(FAILED, startparams,
 			"--async", "reload", "--xxx", "jail")
 		self.assertLogged("Unexpected argument(s) for reload:")
 		self.pruneLog()
@@ -637,16 +637,10 @@ class Fail2banClientTest(Fail2banClientServerBase):
 
 class Fail2banServerTest(Fail2banClientServerBase):
 
-	def execSuccess(self, startparams, *args):
-		self.assertRaises(ExitException, _exec_server,
-			((SERVER,) + startparams + args))
-
-	def execFailed(self, startparams, *args):
-		self.assertRaises(FailExitException, _exec_server,
-			((SERVER,) + startparams + args))
+	exec_command_line = (_exec_server, SERVER,)
 
 	def testServerUsage(self):
-		self.execSuccess((), "-h")
+		self.execCmd(SUCCESS, (), "-h")
 		self.assertLogged("Usage: " + SERVER)
 		self.assertLogged("Report bugs to ")
 
@@ -666,12 +660,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged("Server ready")
 		self.pruneLog()
 		try:
-			self.execSuccess(startparams, "echo", "TEST-ECHO")
-			self.execFailed(startparams, "~~unknown~cmd~failed~~")
+			self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO")
+			self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~")
 		finally:
 			self.pruneLog()
 			# stop:
-			self.execSuccess(startparams, "stop")
+			self.execCmd(SUCCESS, startparams, "stop")
 			self.assertLogged("Shutdown successful")
 			self.assertLogged("Exit with code 0")
 
@@ -682,20 +676,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		startparams = _start_params(tmp, logtarget="INHERITED")
 
 		## wrong config directory
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"-c", pjoin(tmp, "miss"))
 		self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist")
 		self.pruneLog()
 
 		## wrong socket
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock"))
 		self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file")
 		self.pruneLog()
 
 		## already exists:
 		open(pjoin(tmp, "f2b.sock"), 'a').close()
-		self.execFailed((),
+		self.execCmd(FAILED, (),
 			"-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"))
 		self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 		self.pruneLog()
@@ -710,7 +704,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# test configuration is correct:
 		self.pruneLog("[test-phase 0]")
-		self.execSuccess(startparams, "--test")
+		self.execCmd(SUCCESS, startparams, "--test")
 		self.assertLogged("OK: configuration test is successful")
 
 		# append one wrong configured jail:
@@ -719,14 +713,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# first try test config:
 		self.pruneLog("[test-phase 0a]")
-		self.execFailed(startparams, "--test")
+		self.execCmd(FAILED, startparams, "--test")
 		self.assertLogged("Unable to read the filter 'broken-jail-filter'",
 			"Errors in jail 'broken-jail'.",
 			"ERROR: test configuration failed", all=True)
 
 		# failed to start with test config:
 		self.pruneLog("[test-phase 0b]")
-		self.execFailed(startparams, "-t", "start")
+		self.execCmd(FAILED, startparams, "-t", "start")
 		self.assertLogged("Unable to read the filter 'broken-jail-filter'",
 			"Errors in jail 'broken-jail'.",
 			"ERROR: test configuration failed", all=True)
@@ -857,7 +851,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.pruneLog("[test-phase 1a]")
 		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test1log)
-		self.execSuccess(startparams, "reload")
+		self.execCmd(SUCCESS, startparams, "reload")
 		self.assertLogged(
 			"Reload finished.",
 			"1 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME)
@@ -886,7 +880,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_file(test1log, "w+")
 		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(test1log)
-		self.execSuccess(startparams, "reload")
+		self.execCmd(SUCCESS, startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test not unbanned / banned again:
 		self.assertNotLogged(
@@ -918,7 +912,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			start= "               echo '[<name>] %s: started.'" % "test-action1",
 			reload="               echo '[<name>] %s: reloaded.'" % "test-action1", 
 			stop=  "               echo '[<name>] %s: stopped.'" % "test-action1")
-		self.execSuccess(startparams, "reload")
+		self.execCmd(SUCCESS, startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test not unbanned / banned again:
 		self.assertNotLogged(
@@ -979,7 +973,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# restart jail without unban all:
 		self.pruneLog("[test-phase 2c]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"restart", "test-jail2")
 		self.assertLogged(
 			"Reload finished.",
@@ -1007,9 +1001,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# ban manually to test later flush by unban all:
 		self.pruneLog("[test-phase 2d]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"set", "test-jail2", "banip", "192.0.2.21")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"set", "test-jail2", "banip", "192.0.2.22")
 		self.assertLogged(
 			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22",
@@ -1017,7 +1011,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# restart jail with unban all:
 		self.pruneLog("[test-phase 2e]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"restart", "--unban", "test-jail2")
 		self.assertLogged(
 			"Reload finished.",
@@ -1050,7 +1044,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
-		self.execSuccess(startparams, "reload", "test-jail1")
+		self.execCmd(SUCCESS, startparams, "reload", "test-jail1")
 		self.assertLogged(
 			"Reload finished.", all=True, wait=MID_WAITTIME)
 		self.assertLogged(
@@ -1065,7 +1059,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# whole reload, but this time with jail1 only (jail2 should be stopped via configuration):
 		self.pruneLog("[test-phase 4]")
 		_write_jail_cfg(enabled=[1])
-		self.execSuccess(startparams, "reload")
+		self.execCmd(SUCCESS, startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# test both jails should be reloaded:
 		self.assertLogged(
@@ -1102,7 +1096,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# unban single ips:
 		self.pruneLog("[test-phase 6]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"--async", "unban", "192.0.2.5", "192.0.2.6")
 		self.assertLogged(
 			"192.0.2.5 is not banned",
@@ -1111,7 +1105,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# reload all (one jail) with unban all:
 		self.pruneLog("[test-phase 7]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"reload", "--unban")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		# reloads unbanned all:
@@ -1134,7 +1128,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# unban all (just to test command, already empty - nothing to unban):
 		self.pruneLog("[test-phase 7b]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"--async", "unban", "--all")
 		self.assertLogged(
 			"Flush ban list",
@@ -1143,7 +1137,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# backend-switch (restart instead of reload):
 		self.pruneLog("[test-phase 8a]")
 		_write_jail_cfg(enabled=[1], backend="xxx-unknown-backend-zzz")
-		self.execFailed(startparams, "reload")
+		self.execCmd(FAILED, startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)
 		self.assertLogged(
 			"Restart jail 'test-jail1' (reason: 'polling' != ", 
@@ -1151,18 +1145,18 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		self.pruneLog("[test-phase 8b]")
 		_write_jail_cfg(enabled=[1])
-		self.execSuccess(startparams, "reload")
+		self.execCmd(SUCCESS, startparams, "reload")
 		self.assertLogged("Reload finished.", all=True, wait=MID_WAITTIME)	
 
 		# several small cases (cover several parts):
 		self.pruneLog("[test-phase end-1]")
 		# wrong jail (not-started):
-		self.execFailed(startparams,
+		self.execCmd(FAILED, startparams,
 			"--async", "reload", "test-jail2")
 		self.assertLogged("the jail 'test-jail2' does not exist")
 		self.pruneLog()
 		# unavailable jail (but exit 0), using --if-exists option:
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"--async", "reload", "--if-exists", "test-jail2")
 		self.assertNotLogged(
 			"Creating new jail 'test-jail2'",
@@ -1170,7 +1164,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 		# restart all jails (without restart server):
 		self.pruneLog("[test-phase end-2]")
-		self.execSuccess(startparams,
+		self.execCmd(SUCCESS, startparams,
 			"--async", "reload", "--restart", "--all")
 		self.assertLogged(
 			"Jail 'test-jail1' stopped", 
@@ -1215,7 +1209,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[nginx-blck-lst] Ban 125-000-003",
 			"[nginx-blck-lst] Ban 125-000-004",
 			"[nginx-blck-lst] Ban 125-000-005",
-			"Banned 5 / 5, 5 ticket(s)",
+			"5 ticket(s)",
 			all=True, wait=MID_WAITTIME
 		)
 		_out_file(mpfn)
@@ -1227,7 +1221,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertIn('\\125-000-005 1;\n', mp)
 
 		# unban 1, 2 and 5:
-		self.execSuccess(startparams, 'unban', '125-000-001', '125-000-002', '125-000-005')
+		self.execCmd(SUCCESS, startparams, 'unban', '125-000-001', '125-000-002', '125-000-005')
 		_out_file(mpfn)
 		# check really unbanned but other sessions are still present (blacklisted in map-file):
 		mp = _read_file(mpfn)
@@ -1238,8 +1232,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertIn('\\125-000-004 1;\n', mp)
 
 		# stop server and wait for end:
-		self.execSuccess(startparams, 'stop')
-		self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True, wait=MID_WAITTIME)
+		self.execCmd(SUCCESS, startparams, 'stop')
+		self.waitForServerEnd(SUCCESS)
 
 		# check flushed (all sessions were deleted from map-file):
 		self.assertLogged("[nginx-blck-lst] Flush ticket(s) with nginx-block-map")

From a1fd2c507e68d02fd30f707f6af7e302e629aebf Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 13:00:29 +0100
Subject: [PATCH 587/623] method `waitForServerEnd` renamed into
 `stopAndWaitForServerEnd` (because will also stop the server)

---
 fail2ban/tests/fail2banclienttestcase.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 11d1983e..1e2d1b33 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -304,7 +304,7 @@ def with_foreground_server_thread(startextra={}):
 				th.start()
 				# to wait for end of server, default accept any exit code, because multi-threaded, 
 				# thus server can exit in-between...
-				def _waitForServerEnd(code=(SUCCESS, FAILED)):
+				def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)):
 					# if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state):
 					if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")):
 						Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME)
@@ -315,7 +315,7 @@ def with_foreground_server_thread(startextra={}):
 						Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
 						self.assertTrue(phase.get('end', None))
 						self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True)
-				self.waitForServerEnd = _waitForServerEnd
+				self.stopAndWaitForServerEnd = _stopAndWaitForServerEnd
 				# wait for start thread:
 				Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME)
 				self.assertTrue(phase.get('start', None))
@@ -330,12 +330,12 @@ def with_foreground_server_thread(startextra={}):
 					# wait for server end (if not yet already exited):
 					DefLogSys.info('=== within server: end.  ===')
 					self.pruneLog()
-					self.waitForServerEnd()
+					self.stopAndWaitForServerEnd()
 					# we start client/server directly in current process (new thread),
 					# so don't kill (same process) - if success, just wait for end of worker:
 					if phase.get('end', None):
 						th.join()
-				self.waitForServerEnd = None
+				self.stopAndWaitForServerEnd = None
 		return wrapper
 	return _deco_wrapper
 
@@ -1232,8 +1232,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertIn('\\125-000-004 1;\n', mp)
 
 		# stop server and wait for end:
-		self.execCmd(SUCCESS, startparams, 'stop')
-		self.waitForServerEnd(SUCCESS)
+		self.stopAndWaitForServerEnd(SUCCESS)
 
 		# check flushed (all sessions were deleted from map-file):
 		self.assertLogged("[nginx-blck-lst] Flush ticket(s) with nginx-block-map")

From 80932af4062b169341b1de581d9a781435546f0c Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 13:29:35 +0100
Subject: [PATCH 588/623] coverage: testErrorsInLoop should avoid sporadic
 coverage changes, if some communication errors not occurred sometimes.

---
 fail2ban/server/asyncserver.py   |  4 +++-
 fail2ban/tests/sockettestcase.py | 19 +++++++++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index 11c81649..e254979d 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -122,7 +122,9 @@ def loop(active, timeout=None, use_poll=False):
 	if timeout is None:
 		timeout = Utils.DEFAULT_SLEEP_TIME
 	poll = asyncore.poll
-	if use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover
+	if callable(use_poll):
+		poll = use_poll
+	elif use_poll and asyncore.poll2 and hasattr(asyncore.select, 'poll'): # pragma: no cover
 		logSys.debug('Server listener (select) uses poll')
 		# poll2 expected a timeout in milliseconds (but poll and loop in seconds):
 		timeout = float(timeout) / 1000
diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py
index 1a94a952..4f9b9d7a 100644
--- a/fail2ban/tests/sockettestcase.py
+++ b/fail2ban/tests/sockettestcase.py
@@ -31,8 +31,10 @@ import threading
 import time
 import unittest
 
+from .utils import LogCaptureTestCase
+
 from .. import protocol
-from ..server.asyncserver import AsyncServer, AsyncServerException
+from ..server.asyncserver import AsyncServer, AsyncServerException, loop
 from ..server.utils import Utils
 from ..client.csocket import CSocket
 
@@ -126,7 +128,20 @@ class Socket(unittest.TestCase):
 		self.assertFalse(os.path.exists(self.sock_name))
 
 
-class ClientMisc(unittest.TestCase):
+class ClientMisc(LogCaptureTestCase):
+
+	def testErrorsInLoop(self):
+		phase = {'cntr': 0}
+		def _active():
+			return phase['cntr'] < 40
+		def _poll(*args):
+			phase['cntr'] += 1
+			raise Exception('test *%d*' % phase['cntr'])
+		# test errors "catched" and logged:
+		loop(_active, use_poll=_poll)
+		self.assertLogged("test *1*", "test *10*", "test *20*", all=True)
+		self.assertLogged("Too many errors - stop logging connection errors")
+		self.assertNotLogged("test *21*", "test *22*", "test *23*", all=True)
 
 	def testPrintFormattedAndWiki(self):
 		# redirect stdout to devnull

From a10d544ddc21b4f9ccf238b3f582ddf44747e373 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 14:12:19 +0100
Subject: [PATCH 589/623] coverage: fix another sporadic coverage decrease, if
 idle mode never reached in some test-cases (e. g. by slowly reloading of
 jails).

---
 fail2ban/server/actions.py       |  4 +++-
 fail2ban/tests/actiontestcase.py | 19 ++++++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index 73042434..f98c6682 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -276,8 +276,10 @@ class Actions(JailThread, Mapping):
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
 			if self.idle:
+				logSys.debug("Actions: enter idle mode")
 				Utils.wait_for(lambda: not self.active or not self.idle,
-					self.sleeptime * 10, self.sleeptime)
+					lambda: False, self.sleeptime)
+				logSys.debug("Actions: leave idle mode")
 				continue
 			if not Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime):
 				self.__checkUnBan()
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index cbd0aaca..4c92057b 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -30,9 +30,10 @@ import time
 import unittest
 
 from ..server.action import CommandAction, CallingMap, substituteRecursiveTags
-from ..server.actions import OrderedDict
+from ..server.actions import OrderedDict, Actions
 from ..server.utils import Utils
 
+from .dummyjail import DummyJail
 from .utils import LogCaptureTestCase
 from .utils import pid_exists
 
@@ -568,3 +569,19 @@ class CommandActionTest(LogCaptureTestCase):
 		self.assertIn("'b': 11", s)
 		self.assertIn("'c': ", s) # presents as callable
 		self.assertNotIn("'c': ''", s) # but not empty
+
+	def testActionsIdleMode(self):
+		a = Actions(DummyJail())
+		a.sleeptime = 0.0001;   # don't need to wait long
+		# enter idle mode right now (start idle):
+		a.idle = True;
+		# start:
+		a.start()
+		# wait for enter/leave of idle mode:
+		self.assertLogged("Actions: enter idle mode", wait=10)
+		# leave idle mode:
+		a.idle = False
+		self.assertLogged("Actions: leave idle mode", wait=10)
+		# stop it:
+		a.active = False
+		a.join()
\ No newline at end of file

From ab3d03beec11b34447545a2359d65bc39d3c6a04 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 16:13:57 +0100
Subject: [PATCH 590/623] Better variant of repair database: recreate all
 tables/indices, that can be missing after supposedly successful rescue

---
 fail2ban/server/database.py | 85 ++++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 39 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 5474311b..f301fdf8 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -129,14 +129,15 @@ class Fail2BanDb(object):
 	purgeage
 	"""
 	__version__ = 2
-	# Note all _TABLE_* strings must end in ';' for py26 compatibility
-	_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
-	_TABLE_jails = "CREATE TABLE jails(" \
+	# Note all SCRIPTS strings must end in ';' for py26 compatibility
+	_CREATE_SCRIPTS = (
+		 ('fail2banDb', "CREATE TABLE IF NOT EXISTS fail2banDb(version INTEGER);")
+		,('jails', "CREATE TABLE IF NOT EXISTS jails(" \
 			"name TEXT NOT NULL UNIQUE, " \
 			"enabled INTEGER NOT NULL DEFAULT 1" \
 			");" \
-			"CREATE INDEX jails_name ON jails(name);"
-	_TABLE_logs = "CREATE TABLE logs(" \
+			"CREATE INDEX IF NOT EXISTS jails_name ON jails(name);")
+		,('logs', "CREATE TABLE IF NOT EXISTS logs(" \
 			"jail TEXT NOT NULL, " \
 			"path TEXT, " \
 			"firstlinemd5 TEXT, " \
@@ -145,22 +146,24 @@ class Fail2BanDb(object):
 			"UNIQUE(jail, path)," \
 			"UNIQUE(jail, path, firstlinemd5)" \
 			");" \
-			"CREATE INDEX logs_path ON logs(path);" \
-			"CREATE INDEX logs_jail_path ON logs(jail, path);"
+			"CREATE INDEX IF NOT EXISTS logs_path ON logs(path);" \
+			"CREATE INDEX IF NOT EXISTS logs_jail_path ON logs(jail, path);")
 			#TODO: systemd journal features \
 			#"journalmatch TEXT, " \
 			#"journlcursor TEXT, " \
-			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available \
-	_TABLE_bans = "CREATE TABLE bans(" \
+			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
+		,('bans', "CREATE TABLE IF NOT EXISTS bans(" \
 			"jail TEXT NOT NULL, " \
 			"ip TEXT, " \
 			"timeofban INTEGER NOT NULL, " \
 			"data JSON, " \
 			"FOREIGN KEY(jail) REFERENCES jails(name) " \
 			");" \
-			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban);" \
-			"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
-			"CREATE INDEX bans_ip ON bans(ip);" \
+			"CREATE INDEX IF NOT EXISTS bans_jail_timeofban_ip ON bans(jail, timeofban);" \
+			"CREATE INDEX IF NOT EXISTS bans_jail_ip ON bans(jail, ip);" \
+			"CREATE INDEX IF NOT EXISTS bans_ip ON bans(ip);")
+	)
+	_CREATE_TABS = dict(_CREATE_SCRIPTS)
 
 
 	def __init__(self, filename, purgeAge=24*60*60):
@@ -206,16 +209,9 @@ class Fail2BanDb(object):
 			# speedup: temporary tables and indices are kept in memory:
 			cur.execute("PRAGMA temp_store = MEMORY")
 
-			if checkIntegrity:
-				logSys.debug("  Check integrity ...")
-				cur.execute("PRAGMA integrity_check")
-				for s in cur.fetchall():
-					logSys.debug("  %s", s)
-				self._db.commit()
-
 			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		except sqlite3.OperationalError:
-			logSys.warning("New database created. Version '%i'",
+			logSys.warning("New database created. Version '%r'",
 				self.createDb())
 		except sqlite3.Error as e:
 			logSys.error(
@@ -233,14 +229,23 @@ class Fail2BanDb(object):
 			if version < Fail2BanDb.__version__:
 				newversion = self.updateDb(version)
 				if newversion == Fail2BanDb.__version__:
-					logSys.warning( "Database updated from '%i' to '%i'",
+					logSys.warning( "Database updated from '%r' to '%r'",
 						version, newversion)
 				else: # pragma: no cover
-					logSys.error( "Database update failed to achieve version '%i'"
-						": updated from '%i' to '%i'",
+					logSys.error( "Database update failed to achieve version '%r'"
+						": updated from '%r' to '%r'",
 						Fail2BanDb.__version__, version, newversion)
 					raise RuntimeError('Failed to fully update')
 		finally:
+			if checkIntegrity:
+				logSys.debug("  Create missing tables/indices ...")
+				self._createDb(cur, incremental=True)
+				logSys.debug("  -> ok")
+				logSys.debug("  Check integrity ...")
+				cur.execute("PRAGMA integrity_check")
+				for s in cur.fetchall():
+					logSys.debug("  -> %s", ' '.join(s))
+				self._db.commit()
 			if cur:
 				# pypy: set journal mode after possible upgrade db:
 				if pypy:
@@ -261,6 +266,8 @@ class Fail2BanDb(object):
 			return self.__dbBackupFilename
 	
 	def repairDB(self):
+		class RepairException(Exception):
+			pass
 		# avoid endless recursion if reconnect failed again for some reasons:
 		_repairDB = self.repairDB
 		self.repairDB = None
@@ -280,13 +287,14 @@ class Fail2BanDb(object):
 				self._connectDB(checkIntegrity=True)
 			else:
 				logSys.info("  Repair seems to be failed, restored %d byte(s).", dbFileSize)
-				raise Exception('Recreate ...')
+				raise RepairException('Recreate ...')
 		except Exception as e:
 			# if still failed, just recreate database as fallback:
 			logSys.error("  Error repairing of fail2ban database '%s': %s",
-				self._dbFilename, e.args[0])
+				self._dbFilename, e.args[0], 
+				exc_info=(not isinstance(e, RepairException) and logSys.getEffectiveLevel() <= 10))
 			os.remove(self._dbFilename)
-			self._connectDB()
+			self._connectDB(checkIntegrity=True)
 		finally:
 			self.repairDB = _repairDB
 
@@ -306,24 +314,23 @@ class Fail2BanDb(object):
 	def purgeage(self, value):
 		self._purgeAge = MyTime.str2seconds(value)
 
-	@commitandrollback
-	def createDb(self, cur):
+	def _createDb(self, cur, incremental=False):
 		"""Creates a new database, called during initialisation.
 		"""
-		# Version info
-		cur.executescript(Fail2BanDb._TABLE_fail2banDb)
-		cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
+		# create all (if not exists):
+		for (n, s) in Fail2BanDb._CREATE_SCRIPTS:
+			cur.executescript(s)
+		# save current database version (if not already set):			
+		cur.execute("INSERT INTO fail2banDb(version)"
+			" SELECT ? WHERE NOT EXISTS (SELECT 1 FROM fail2banDb LIMIT 1)",
 			(Fail2BanDb.__version__, ))
-		# Jails
-		cur.executescript(Fail2BanDb._TABLE_jails)
-		# Logs
-		cur.executescript(Fail2BanDb._TABLE_logs)
-		# Bans
-		cur.executescript(Fail2BanDb._TABLE_bans)
-
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
+	@commitandrollback
+	def createDb(self, cur, incremental=False):
+		return self._createDb(cur, incremental);
+
 	@commitandrollback
 	def updateDb(self, cur, version):
 		"""Update an existing database, called during initialisation.
@@ -347,7 +354,7 @@ class Fail2BanDb(object):
 						"INSERT INTO logs SELECT * from logs_temp;"
 						"DROP TABLE logs_temp;"
 						"UPDATE fail2banDb SET version = 2;"
-						"COMMIT;" % Fail2BanDb._TABLE_logs)
+						"COMMIT;" % Fail2BanDb._CREATE_TABS['logs'])
 
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]

From 1e39c2600cc2306329aef9bb5ddddf4a27e48fad Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 22 Dec 2017 17:21:11 +0100
Subject: [PATCH 591/623] cherry-pick from 0.11: changes in updateDb because it
 can be executed after repair, and some tables can be missing.

---
 fail2ban/server/database.py        | 30 +++++++++++++++++++++---------
 fail2ban/tests/databasetestcase.py |  2 +-
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index f301fdf8..5e23fedc 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -331,6 +331,12 @@ class Fail2BanDb(object):
 	def createDb(self, cur, incremental=False):
 		return self._createDb(cur, incremental);
 
+	def _tableExists(self, cur, table):
+		cur.execute("select 1 where exists ("
+			"select 1 from sqlite_master WHERE type='table' AND name=?)", (table,))
+		res = cur.fetchone()
+		return res is not None and res[0]
+
 	@commitandrollback
 	def updateDb(self, cur, version):
 		"""Update an existing database, called during initialisation.
@@ -340,14 +346,14 @@ class Fail2BanDb(object):
 		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
 						"Attempt to travel to future version of database ...how did you get here??")
+		try:
+			logSys.info("Upgrade database: %s from version '%r'", self._dbBackupFilename, version)
+			if not os.path.isfile(self._dbBackupFilename):
+				shutil.copyfile(self.filename, self._dbBackupFilename)
+				logSys.info("  Database backup created: %s", self._dbBackupFilename)
 
-		logSys.info("Uprade database: %s", self._dbBackupFilename)
-		if not os.path.isfile(self._dbBackupFilename):
-			shutil.copyfile(self.filename, self._dbBackupFilename)
-			logSys.info("  Database backup created: %s", self._dbBackupFilename)
-
-		if version < 2:
-			cur.executescript("BEGIN TRANSACTION;"
+			if version < 2 and self._tableExists(cur, "logs"):
+				cur.executescript("BEGIN TRANSACTION;"
 						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
 						"DROP TABLE logs;"
 						"%s;"
@@ -356,8 +362,14 @@ class Fail2BanDb(object):
 						"UPDATE fail2banDb SET version = 2;"
 						"COMMIT;" % Fail2BanDb._CREATE_TABS['logs'])
 
-		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
-		return cur.fetchone()[0]
+			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+			return cur.fetchone()[0]
+		except Exception as e:
+			# if still failed, just recreate database as fallback:
+			logSys.error("Failed to upgrade database '%s': %s",
+				self._dbFilename, e.args[0], 
+				exc_info=logSys.getEffectiveLevel() <= 10)
+			raise
 
 	@commitandrollback
 	def addJail(self, cur, jail):
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 5ac590f5..5124d75e 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -144,7 +144,7 @@ class DatabaseTest(LogCaptureTestCase):
 					self.assertEqual(len(self.db.getJailNames()), 1)
 				else: # recreated:
 					self.assertLogged("Repair seems to be failed",
-						"New database created.", all=True)
+						"Check integrity", "New database created.", all=True)
 					self.assertEqual(len(self.db.getLogPaths()), 0)
 					self.assertEqual(len(self.db.getJailNames()), 0)
 			finally:

From b28dfb965a373e9a67add2f51b659bc1b95fd944 Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Wed, 3 Jan 2018 18:39:30 +0100
Subject: [PATCH 592/623] Fix filter not catching asterisk requests with quote
 character in username (fixes #2010)

---
 config/filter.d/asterisk.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index e94754c0..5ebbe92b 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -25,7 +25,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
-            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
+            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '.*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 
 # FreePBX (todo: make optional in v.0.10):
 #            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$

From a53ee46ad465230f002b1eb998437fd4ddd7ca3b Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Wed, 3 Jan 2018 18:48:11 +0100
Subject: [PATCH 593/623] add test for asterisk pjsip attack with quote in
 username

---
 fail2ban/tests/files/logs/asterisk | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 837011e0..523631f6 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -80,9 +80,8 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Failed to authenticate
 # failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Error to authenticate
-# Failed authentication with pjsip on Asterisk 13+
-# failJSON: { "time": "2016-06-08T23:40:26", "match": true , "host": "2.3.4.5" }
-[2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found
+# failJSON: { "time": "2017-12-14T22:18:00", "match": true , "host": "1.2.3.4" }
+[2017-12-14 22:18:00] NOTICE[1943] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '<sip:'or''='@1.2.3.4>' failed for '185.107.94.10:43678' (callid: UmOkE9yQPGOsF3Az24YTRe..) - No matching endpoint found
 
 # failJSON: { "time": "2016-06-09T00:01:02", "match": true , "host": "192.0.2.1" }
 [2016-06-09 00:01:02] NOTICE [22382] manager.c: 192.0.2.1 failed to authenticate as 'admin'

From 184202c6aaaa62fd28f3b7cd677b9edc371d4ffd Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Wed, 3 Jan 2018 18:49:38 +0100
Subject: [PATCH 594/623] remove duplicate testcase

---
 fail2ban/tests/files/logs/asterisk | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 523631f6..27535db0 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -92,10 +92,6 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.5" }
 [2016-05-06 07:08:09] NOTICE[6772] manager.c: 192.0.2.5 failed to authenticate as 'Admin'
 
-# PJSip Errors
-# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.6" }
-[2016-05-06 07:08:09] NOTICE[17103] res_pjsip/pjsip_distributor.c: Request from '"test1" <sip:test1@2.3.4.5>' failed for '192.0.2.6:5678' (callid: deadbeef) - No matching endpoint found
-
 # # FreePBX Warnings
 # #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.4" }
 # [2016-05-06 07:08:09] WARNING[6410][C-00000bac] Ext. 50048943556071: Friendly Scanner from 192.0.2.4

From eaf5e88692499fb91adb52a41d6d95add4a37515 Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Wed, 3 Jan 2018 19:00:09 +0100
Subject: [PATCH 595/623] replace actual offenders ip with 1.2.3.4

---
 fail2ban/tests/files/logs/asterisk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 27535db0..f5f4d419 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -81,7 +81,7 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Error to authenticate
 # failJSON: { "time": "2017-12-14T22:18:00", "match": true , "host": "1.2.3.4" }
-[2017-12-14 22:18:00] NOTICE[1943] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '<sip:'or''='@1.2.3.4>' failed for '185.107.94.10:43678' (callid: UmOkE9yQPGOsF3Az24YTRe..) - No matching endpoint found
+[2017-12-14 22:18:00] NOTICE[1943] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '<sip:'or''='@4.5.6.7>' failed for '1.2.3.4:43678' (callid: UmOkE9yQPGOsF3Az24YTRe..) - No matching endpoint found
 
 # failJSON: { "time": "2016-06-09T00:01:02", "match": true , "host": "192.0.2.1" }
 [2016-06-09 00:01:02] NOTICE [22382] manager.c: 192.0.2.1 failed to authenticate as 'admin'

From 94f0b15c320f007a9cd7cce97d12f45318b5553d Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Mon, 8 Jan 2018 14:54:32 +0100
Subject: [PATCH 596/623] Allow faster parsing of hosts without ' characters in
 them

---
 config/filter.d/asterisk.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf
index 5ebbe92b..0cb1b70a 100644
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@@ -25,7 +25,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
             ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
             ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
-            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '.*' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
+            ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '(?:[^']*|.*?)' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
 
 # FreePBX (todo: make optional in v.0.10):
 #            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$

From aab54bb0dde72831fb85660304b0a320e5f8da2c Mon Sep 17 00:00:00 2001
From: Yannik Sembritzki <yannik@sembritzki.me>
Date: Mon, 8 Jan 2018 22:29:43 +0100
Subject: [PATCH 597/623] don't replace normal test case with specialized test
 case

---
 fail2ban/tests/files/logs/asterisk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index f5f4d419..5e68772a 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -80,6 +80,8 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Failed to authenticate
 # failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Error to authenticate
+# failJSON: { "time": "2016-06-08T23:40:26", "match": true , "host": "2.3.4.5" }
+[2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found 
 # failJSON: { "time": "2017-12-14T22:18:00", "match": true , "host": "1.2.3.4" }
 [2017-12-14 22:18:00] NOTICE[1943] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '<sip:'or''='@4.5.6.7>' failed for '1.2.3.4:43678' (callid: UmOkE9yQPGOsF3Az24YTRe..) - No matching endpoint found
 

From 2c69c0e7e5cfa6646afea032eb10278d75cab6fc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 9 Jan 2018 12:15:56 +0100
Subject: [PATCH 598/623] flush jail in database: bulk remove of all IPs in the
 database (e. g. reload --unban).

---
 fail2ban/server/actions.py  | 12 ++++++------
 fail2ban/server/database.py | 24 +++++++++++++++---------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index f98c6682..fd4263c3 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -450,7 +450,7 @@ class Actions(JailThread, Mapping):
 		"""
 		log = True
 		if actions is None:
-			logSys.debug("Flush ban list")
+			logSys.debug("  Flush ban list")
 			lst = self.__banManager.flushBanList()
 		else:
 			log = False # don't log "[jail] Unban ..." if removing actions only.
@@ -465,16 +465,16 @@ class Actions(JailThread, Mapping):
 			else:
 				unbactions[name] = action
 		actions = unbactions
+		# flush the database also:
+		if db and self._jail.database is not None:
+			logSys.debug("  Flush jail in database")
+			self._jail.database.delBan(self._jail)
 		# unban each ticket with non-flasheable actions:
 		for ticket in lst:
-			# delete ip from database also:
-			if db and self._jail.database is not None:
-				ip = str(ticket.getIP())
-				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
 			self.__unBan(ticket, actions=actions, log=log)
 			cnt += 1
-		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
+		logSys.debug("  Unbanned %s, %s ticket(s) in %r", 
 			cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 5e23fedc..d7ba12c1 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -533,20 +533,26 @@ class Fail2BanDb(object):
 				ticket.getData()))
 
 	@commitandrollback
-	def delBan(self, cur, jail, ip):
-		"""Delete a ban from the database.
+	def delBan(self, cur, jail, *args):
+		"""Delete a single or multiple tickets from the database.
 
 		Parameters
 		----------
 		jail : Jail
-			Jail in which the ban has occurred.
-		ip : str
-			IP to be removed.
+			Jail in which the ticket(s) should be removed.
+		args : list of IP
+			IPs to be removed, if not given all tickets of jail will be removed.
 		"""
-		queryArgs = (jail.name, str(ip));
-		cur.execute(
-			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
-			queryArgs);
+		query = "DELETE FROM bans WHERE jail = ?"
+		queryArgs = [jail.name];
+		if not len(args):
+			cur.execute(query, queryArgs);
+			return
+		query += " AND ip = ?"
+		queryArgs.append('');
+		for ip in args:
+			queryArgs[1] = str(ip);
+			cur.execute(query, queryArgs);
 
 	@commitandrollback
 	def _getBans(self, cur, jail=None, bantime=None, ip=None):

From f6d0c86533ac86b8c5723e690577d9265a198c17 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Tue, 9 Jan 2018 12:16:37 +0100
Subject: [PATCH 599/623] test cases extended: flush jail in database

---
 fail2ban/tests/databasetestcase.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index 5124d75e..54d05677 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -243,9 +243,10 @@ class DatabaseTest(LogCaptureTestCase):
 		ticket = FailTicket("127.0.0.1", 0, ["abc\n"])
 		self.db.addBan(self.jail, ticket)
 
-		self.assertEqual(len(self.db.getBans(jail=self.jail)), 1)
+		tickets = self.db.getBans(jail=self.jail)
+		self.assertEqual(len(tickets), 1)
 		self.assertTrue(
-			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
+			isinstance(tickets[0], FailTicket))
 
 	def testAddBanInvalidEncoded(self):
 		if Fail2BanDb is None: # pragma: no cover
@@ -278,10 +279,28 @@ class DatabaseTest(LogCaptureTestCase):
 			or readtickets[2] == tickets[2]
 		)
 
+	def _testAdd3Bans(self):
+		self.testAddJail()
+		for i in (1, 2, 3):
+			ticket = FailTicket(("192.0.2.%d" % i), 0, ["test\n"])
+			self.db.addBan(self.jail, ticket)
+		tickets = self.db.getBans(jail=self.jail)
+		self.assertEqual(len(tickets), 3)
+		return tickets
+
 	def testDelBan(self):
-		self.testAddBan()
-		ticket = self.db.getBans(jail=self.jail)[0]
-		self.db.delBan(self.jail, ticket.getIP())
+		tickets = self._testAdd3Bans()
+		# delete single IP:
+		self.db.delBan(self.jail, tickets[0].getIP())
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 2)
+		# delete two IPs:
+		self.db.delBan(self.jail, tickets[1].getIP(), tickets[2].getIP())
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 0)
+
+	def testFlushBans(self):
+		self._testAdd3Bans()
+		# flush all bans:
+		self.db.delBan(self.jail)
 		self.assertEqual(len(self.db.getBans(jail=self.jail)), 0)
 
 	def testGetBansWithTime(self):

From a1d1498561fdfea527d44c8fc75f69e27b6d5a1f Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 9 Jan 2018 21:13:02 +0100
Subject: [PATCH 600/623] Restore log-entries not affected by #2011

---
 fail2ban/tests/files/logs/asterisk | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk
index 5e68772a..0955cfe7 100644
--- a/fail2ban/tests/files/logs/asterisk
+++ b/fail2ban/tests/files/logs/asterisk
@@ -81,7 +81,7 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # failJSON: { "time": "2016-05-23T10:18:16", "match": true , "host": "1.2.3.4" }
 [2016-05-23 10:18:16] NOTICE[19388] res_pjsip/pjsip_distributor.c: Request from '"1000" <sip:1000@10.0.0.1>' failed for '1.2.3.4:48336' (callid: 276666022) - Error to authenticate
 # failJSON: { "time": "2016-06-08T23:40:26", "match": true , "host": "2.3.4.5" }
-[2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found 
+[2016-06-08 23:40:26] NOTICE[32497] res_pjsip/pjsip_distributor.c: Request from '"317" <sip:317@1.2.3.4>' failed for '2.3.4.5:5089' (callid: 206f178f-896564cb-57573f49@1.2.3.4) - No matching endpoint found
 # failJSON: { "time": "2017-12-14T22:18:00", "match": true , "host": "1.2.3.4" }
 [2017-12-14 22:18:00] NOTICE[1943] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '<sip:'or''='@4.5.6.7>' failed for '1.2.3.4:43678' (callid: UmOkE9yQPGOsF3Az24YTRe..) - No matching endpoint found
 
@@ -94,6 +94,10 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.5" }
 [2016-05-06 07:08:09] NOTICE[6772] manager.c: 192.0.2.5 failed to authenticate as 'Admin'
 
+# PJSip Errors
+# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.6" }
+[2016-05-06 07:08:09] NOTICE[17103] res_pjsip/pjsip_distributor.c: Request from '"test1" <sip:test1@2.3.4.5>' failed for '192.0.2.6:5678' (callid: deadbeef) - No matching endpoint found
+
 # # FreePBX Warnings
 # #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.4" }
 # [2016-05-06 07:08:09] WARNING[6410][C-00000bac] Ext. 50048943556071: Friendly Scanner from 192.0.2.4

From f7e2d3610bb98d4dc01076811bd393b11e8c5f4f Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 9 Jan 2018 21:19:01 +0100
Subject: [PATCH 601/623] Update ChangeLog

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 1010d6d7..1917f079 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -30,6 +30,7 @@ releases.
     - fixed failregex AMI Asterisk authentification failed (see gh-1302)
     - removed invalid (vulnerable) regex blocking IPs using forign data (from header "from")
       thus not the IP-address that really originates the request (see gh-1927)
+    - fixed failregex for the SQL-injection attempts with single-quotes in connect-string (see gh-2011)
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
     - extended to match pam_authenticate failures with "Permission denied" (gh-1897)

From 3d9a112c8f5369102592c752fd7c2bd1cb395ebc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Jan 2018 10:55:19 +0100
Subject: [PATCH 602/623] cherry-pick newer version of extractOptions, in order
 to avoid large discrepancy between 0.10 and 0.9 config-parsers: allow to use
 dual parameter lists (coming through substitutions), e. g.: `name[p1=0,
 p2="..."][p3='...']`; simplified explanation: `][` treats as `,` in new
 version. cherry-picked from 0.10.

---
 fail2ban/client/jailreader.py | 27 ++++-----------------------
 fail2ban/helpers.py           | 29 +++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 5725f606..fcaffe21 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -27,14 +27,13 @@ __license__ = "GPL"
 import glob
 import json
 import os.path
-import re
 
 from .configreader import ConfigReaderUnshared, ConfigReader
 from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..version import version
 from ..helpers import getLogger
-from ..helpers import splitwords
+from ..helpers import extractOptions, splitwords
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -42,10 +41,6 @@ logSys = getLogger(__name__)
 
 class JailReader(ConfigReader):
 	
-	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
-	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
-	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
 		self.__name = name
@@ -121,7 +116,7 @@ class JailReader(ConfigReader):
 		if self.isEnabled():
 			# Read filter
 			if self.__opts["filter"]:
-				filterName, filterOpt = JailReader.extractOptions(
+				filterName, filterOpt = extractOptions(
 					self.__opts["filter"])
 				self.__filter = FilterReader(
 					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
@@ -150,7 +145,7 @@ class JailReader(ConfigReader):
 				try:
 					if not act:			  # skip empty actions
 						continue
-					actName, actOpt = JailReader.extractOptions(act)
+					actName, actOpt = extractOptions(act)
 					if actName.endswith(".py"):
 						self.__actions.append([
 							"set",
@@ -244,18 +239,4 @@ class JailReader(ConfigReader):
 		stream.insert(0, ["add", self.__name, backend])
 		return stream
 	
-	@staticmethod
-	def extractOptions(option):
-		match = JailReader.optionCRE.match(option)
-		if not match:
-			# TODO proper error handling
-			return None, None
-		option_name, optstr = match.groups()
-		option_opts = dict()
-		if optstr:
-			for optmatch in JailReader.optionExtractRE.finditer(optstr):
-				opt = optmatch.group(1)
-				value = [
-					val for val in optmatch.group(2,3,4) if val is not None][0]
-				option_opts[opt.strip()] = value.strip()
-		return option_name, option_opts
+JailReader.extractOptions = staticmethod(extractOptions)
diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py
index aef39835..4443a672 100644
--- a/fail2ban/helpers.py
+++ b/fail2ban/helpers.py
@@ -137,3 +137,32 @@ def splitwords(s):
 	if not s:
 		return []
 	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
+
+
+#
+# Following function used for parse options from parameter (e.g. `name[p1=0, p2="..."][p3='...']`).
+#
+
+# regex, to extract list of options:
+OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
+# regex, to iterate over single option in option list, syntax:
+# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
+# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
+# `action = act[p1=...][p2=...]`
+OPTION_EXTRACT_CRE = re.compile(
+	r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
+
+def extractOptions(option):
+	match = OPTION_CRE.match(option)
+	if not match:
+		# TODO proper error handling
+		return None, None
+	option_name, optstr = match.groups()
+	option_opts = dict()
+	if optstr:
+		for optmatch in OPTION_EXTRACT_CRE.finditer(optstr):
+			opt = optmatch.group(1)
+			value = [
+				val for val in optmatch.group(2,3,4) if val is not None][0]
+			option_opts[opt.strip()] = value.strip()
+	return option_name, option_opts

From c190631f8842b884199354f34f0ad8caf02051a2 Mon Sep 17 00:00:00 2001
From: Danila Vershinin <ciapnz@gmail.com>
Date: Thu, 5 Oct 2017 18:26:11 +0300
Subject: [PATCH 603/623] New ban action firewallcmd-ipset-allports. Closes
 #1167

---
 ChangeLog                                     |  1 +
 .../action.d/firewallcmd-ipset-allports.conf  | 51 +++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 config/action.d/firewallcmd-ipset-allports.conf

diff --git a/ChangeLog b/ChangeLog
index 1917f079..1cdb28b7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -47,6 +47,7 @@ releases.
 * filter.d/kerio.conf - filter extended with new rules (see gh-1455)
 * filter.d/phpmyadmin-syslog.conf - new filter for phpMyAdmin using syslog for auth logging
 * filter.d/zoneminder.conf - new filter for ZoneMinder (gh-1376)
+* action.d/firewallcmd-ipset-allports.conf - new firewallcmd-ipset-allports action (gh-1167)
 
 
 ver. 0.9.7 (2017/05/11) - awaiting-victory
diff --git a/config/action.d/firewallcmd-ipset-allports.conf b/config/action.d/firewallcmd-ipset-allports.conf
new file mode 100644
index 00000000..eb940ee0
--- /dev/null
+++ b/config/action.d/firewallcmd-ipset-allports.conf
@@ -0,0 +1,51 @@
+# Fail2Ban action file for firewall-cmd/ipset
+#
+# This requires:
+# ipset (package: ipset)
+# firewall-cmd (package: firewalld)
+#
+# This is for ipset protocol 6 (and hopefully later) (ipset v6.14).
+# Use ipset -V to see the protocol and version.
+#
+# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels.
+#
+# If you are running on an older kernel you make need to patch in external
+# modules.
+
+[INCLUDES]
+
+before = iptables-common.conf
+
+[Definition]
+
+actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
+              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m set --match-set fail2ban-<name> src -j <blocktype>
+
+actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m set --match-set fail2ban-<name> src -j <blocktype>
+             ipset flush fail2ban-<name>
+             ipset destroy fail2ban-<name>
+
+actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
+
+actionunban = ipset del fail2ban-<name> <ip> -exist
+
+[Init]
+
+# Option:  chain
+# Notes    specifies the iptables chain to which the fail2ban rules should be
+#          added
+# Values:  [ STRING ]
+#
+chain = INPUT_direct
+
+# Option: bantime
+# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
+# Values:  [ NUM ]  Default: 600
+
+bantime = 600
+
+
+# DEV NOTES:
+#
+# Author: Edgar Hoch and Daniel Black
+# firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness
\ No newline at end of file

From 131b94e11e6a1e05655aad31458556ab2529f24d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Jan 2018 10:11:11 +0100
Subject: [PATCH 604/623] firewallcmd-ipset-allports: implemented in
 `action.d/firewallcmd-ipset.conf` now
 (`action.d/firewallcmd-ipset-allports.conf` removed), usage:

banaction = firewallcmd-ipset[actiontype="<allports>"]
---
 .../action.d/firewallcmd-ipset-allports.conf  | 51 -------------------
 config/action.d/firewallcmd-ipset.conf        | 21 +++++++-
 2 files changed, 19 insertions(+), 53 deletions(-)
 delete mode 100644 config/action.d/firewallcmd-ipset-allports.conf

diff --git a/config/action.d/firewallcmd-ipset-allports.conf b/config/action.d/firewallcmd-ipset-allports.conf
deleted file mode 100644
index eb940ee0..00000000
--- a/config/action.d/firewallcmd-ipset-allports.conf
+++ /dev/null
@@ -1,51 +0,0 @@
-# Fail2Ban action file for firewall-cmd/ipset
-#
-# This requires:
-# ipset (package: ipset)
-# firewall-cmd (package: firewalld)
-#
-# This is for ipset protocol 6 (and hopefully later) (ipset v6.14).
-# Use ipset -V to see the protocol and version.
-#
-# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels.
-#
-# If you are running on an older kernel you make need to patch in external
-# modules.
-
-[INCLUDES]
-
-before = iptables-common.conf
-
-[Definition]
-
-actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m set --match-set fail2ban-<name> src -j <blocktype>
-
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m set --match-set fail2ban-<name> src -j <blocktype>
-             ipset flush fail2ban-<name>
-             ipset destroy fail2ban-<name>
-
-actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
-
-actionunban = ipset del fail2ban-<name> <ip> -exist
-
-[Init]
-
-# Option:  chain
-# Notes    specifies the iptables chain to which the fail2ban rules should be
-#          added
-# Values:  [ STRING ]
-#
-chain = INPUT_direct
-
-# Option: bantime
-# Notes:  specifies the bantime in seconds (handled internally rather than by fail2ban)
-# Values:  [ NUM ]  Default: 600
-
-bantime = 600
-
-
-# DEV NOTES:
-#
-# Author: Edgar Hoch and Daniel Black
-# firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness
\ No newline at end of file
diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf
index 38b0f3d3..62b6e7c2 100644
--- a/config/action.d/firewallcmd-ipset.conf
+++ b/config/action.d/firewallcmd-ipset.conf
@@ -19,9 +19,9 @@ before = iptables-common.conf
 [Definition]
 
 actionstart = ipset create fail2ban-<name> hash:ip timeout <bantime>
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 <actiontype> -m set --match-set fail2ban-<name> src -j <blocktype>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -p <protocol> -m multiport --dports <port> -m set --match-set fail2ban-<name> src -j <blocktype>
+actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 <actiontype> -m set --match-set fail2ban-<name> src -j <blocktype>
              ipset flush fail2ban-<name>
              ipset destroy fail2ban-<name>
 
@@ -44,6 +44,23 @@ chain = INPUT_direct
 
 bantime = 600
 
+# Option: actiontype
+# Notes.: defines additions to the blocking rule
+# Values: leave empty to block all attempts from the host
+# Default: Value of the multiport
+actiontype = <multiport>
+
+# Option: allports
+# Notes.: default addition to block all ports
+# Usage.: use in jail config:  banaction = firewallcmd-ipset[actiontype=<allports>]
+#         for all protocols:   banaction = firewallcmd-ipset[actiontype=""]
+allports = -p <protocol>
+
+# Option: multiport
+# Notes.: addition to block access only to specific ports
+# Usage.: use in jail config:  banaction = firewallcmd-ipset[actiontype=<multiport>]
+multiport = -p <protocol> -m multiport --dports <port>
+
 
 # DEV NOTES:
 #

From 029cd5aa24be005f840d35271e31546da61015e5 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Wed, 10 Jan 2018 11:47:59 +0100
Subject: [PATCH 605/623] Update ChangeLog

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 1cdb28b7..49a34e02 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -44,10 +44,10 @@ releases.
 
 ### Enhancements
 * action.d/cloudflare.conf - Cloudflare API v4 implementation (gh-1651)
+* action.d/firewallcmd-ipset.conf - new parameter `actiontype`, provides `allports` capability (gh-1167)
 * filter.d/kerio.conf - filter extended with new rules (see gh-1455)
 * filter.d/phpmyadmin-syslog.conf - new filter for phpMyAdmin using syslog for auth logging
 * filter.d/zoneminder.conf - new filter for ZoneMinder (gh-1376)
-* action.d/firewallcmd-ipset-allports.conf - new firewallcmd-ipset-allports action (gh-1167)
 
 
 ver. 0.9.7 (2017/05/11) - awaiting-victory

From c36fbdf743c94fbc6bf5cc4da58a1d7298a94809 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Jan 2018 12:13:07 +0100
Subject: [PATCH 606/623] test cases extended in order to cover
 `firewallcmd-ipset` with `allports`

---
 fail2ban/tests/servertestcase.py | 40 +++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 39601db9..a5e93d2d 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1645,7 +1645,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`firewall-cmd --direct --remove-rule ipv6 filter f2b-j-w-fwcmd-ap 0 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
 					),					
 				}),
-				# firewallcmd-ipset --
+				# firewallcmd-ipset (multiport) --
 				('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', {
 					'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',),
 					'ip4-start': (
@@ -1683,6 +1683,44 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`",
 					),					
 				}),
+				# firewallcmd-ipset (allports) --
+				('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=<allports>, protocol="tcp", chain="<known/chain>"]', {
+					'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',),
+					'ip4-start': (
+						"`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`",
+						"`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+					), 
+					'ip6-start': (
+						"`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`",
+						"`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'flush': (
+						"`ipset flush f2b-j-w-fwcmd-ipset-ap`",
+						"`ipset flush f2b-j-w-fwcmd-ipset-ap6`",
+					),
+					'stop': (
+						"`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset-ap`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset-ap`",
+						"`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-fwcmd-ipset-ap6`",
+						"`ipset destroy f2b-j-w-fwcmd-ipset-ap6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`",
+					),					
+				}),
 			)
 			server = TestServer()
 			transm = server._Server__transm

From 314e402fe0fd46b102d94000ec626b212bdca657 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Jan 2018 14:48:25 +0100
Subject: [PATCH 607/623] filter.d/sendmail-auth.conf - extended daemon for
 Fedora 24/RHEL - the daemon name is "sendmail" (gh-1632)

---
 ChangeLog                               | 1 +
 config/filter.d/sendmail-auth.conf      | 2 +-
 fail2ban/tests/files/logs/sendmail-auth | 4 ++++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 49a34e02..ec11539f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -39,6 +39,7 @@ releases.
     - fixed failregex of "AUTH command used when not advertised" to better handle the foreign
       input SMTP command (lower/mixed case auth command, prevent injection) (gh-1979)
 * filter.d/postfix-*.conf - added optional port regex (gh-1902)
+* filter.d/sendmail-auth.conf - extended daemon for Fedora 24/RHEL - the daemon name is "sendmail" (gh-1632)
 
 ### New Features
 
diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf
index 7886e60d..3e2d34a8 100644
--- a/config/filter.d/sendmail-auth.conf
+++ b/config/filter.d/sendmail-auth.conf
@@ -7,7 +7,7 @@ before = common.conf
 
 [Definition]
 
-_daemon = (?:sm-(mta|acceptingconnections))
+_daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
 
 failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
 
diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth
index 7cc18c70..835508f6 100644
--- a/fail2ban/tests/files/logs/sendmail-auth
+++ b/fail2ban/tests/files/logs/sendmail-auth
@@ -10,3 +10,7 @@ Feb 24 12:10:15 kismet sm-acceptingconnections[32053]: s1OHA28u032053: 211-75-6-
 
 # failJSON: { "time": "2005-02-24T13:00:17", "match": true , "host": "95.70.241.192" }
 Feb 24 13:00:17 kismet sm-acceptingconnections[1499]: s1OHxxSn001499: 192.241.70.95.dsl.static.turk.net [95.70.241.192] (may be forged): possible SMTP attack: command=AUTH, count=6
+
+# gh-1632, Fedora 24/RHEL - the daemon name is "sendmail":
+# failJSON: { "time": "2005-02-24T14:00:00", "match": true , "host": "192.0.2.1" }
+Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SMTP attack: command=AUTH, count=5

From 2112145eb4a259f1a40a3ce5b709adfa91cb6d5d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 10 Jan 2018 19:07:20 +0100
Subject: [PATCH 608/623] stop ban of legitimate users with multiple public
 keys (e. g. git, etc), thereby differentiate between "invalid user" (going
 banned earlier) and valid users with public keys, for which the rejects of
 not valid public keys (failures) will be retarded up to "Too many
 authentication failures" resp. disconnect without success (accepted public
 key).

---
 config/filter.d/sshd.conf                     |  4 ++-
 .../filter.d/zzz-sshd-obsolete-multiline.conf |  3 ++-
 fail2ban/tests/files/logs/sshd                | 27 +++++++++++++++++--
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
index 0f9a32ed..ab5fd385 100644
--- a/config/filter.d/sshd.conf
+++ b/config/filter.d/sshd.conf
@@ -34,7 +34,8 @@ prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONT
 
 cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
             ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
-            ^Failed \S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            ^Failed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
             ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
             ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
             ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
@@ -50,6 +51,7 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER>
             ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
             ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>: 11:
             ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by <HOST>%(__suff)s$
+            ^<F-MLFFORGET><F-NOFAIL>Accepted publickey</F-NOFAIL></F-MLFFORGET> for \S+ from <HOST>(?:\s|$)
 
 mdre-normal =
 
diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
index 5a3c44e3..f193009f 100644
--- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
+++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf
@@ -31,7 +31,8 @@ __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
 
 cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)sFailed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+         ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
          ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
          ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__on_port_opt)s\s*$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd
index ce0a2426..b0b760ff 100644
--- a/fail2ban/tests/files/logs/sshd
+++ b/fail2ban/tests/files/logs/sshd
@@ -141,10 +141,33 @@ Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from
 # failJSON: { "time": "2004-11-11T08:04:52", "match": true , "host": "127.0.0.1", "desc": "More complex injecting on auth-info ssh test@localhost, auth-info: ' from 10.10.1.2 port 55555 ssh2'" }
 Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from 127.0.0.1 port 58946 ssh2: from 10.10.1.2 port 55555 ssh2
 
+# Failure on connect of invalid user with public keys:
 # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." }
-Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
+Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for invalid user graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
 # failJSON: { "time": "2005-07-05T18:22:45", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Failed publickey for ..." }
-Jul 05 18:22:45 mercury sshd[4670]: Failed publickey for graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
+Jul 05 18:22:45 mercury sshd[4670]: Failed publickey for invalid user graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI
+
+# Ignore tries of legitimate users with multiple public keys (gh-1263):
+# failJSON: { "match": false }
+Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
+# failJSON: { "match": false }
+Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
+# failJSON: { "match": false }
+Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth]
+# failJSON: { "match": false }
+Nov 28 09:16:03 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
+# failJSON: { "match": false, "desc": "Should be forgotten by success/accepted public key" }
+Nov 28 09:16:03 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]
+
+# Failure on connect with valid user-name but wrong public keys (retarded to disconnect/too many errors, because of gh-1263):
+# failJSON: { "match": false }
+Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
+# failJSON: { "match": false }
+Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
+# failJSON: { "match": false }
+Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth]
+# failJSON: { "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" }
+Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth]
 
 # failJSON: { "match": false }
 Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth]

From 1c0fc73e48ab006cc3f5bd4ac10b8540b450149d Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Jan 2018 10:27:38 +0100
Subject: [PATCH 609/623] Update ChangeLog

---
 ChangeLog | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 47562060..b40b0a0e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -50,9 +50,11 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
   - fixed syntax error in achnor definition (documentation, see gh-1919);
   - enclose ports in braces for multiport jails (see gh-1925);
 * `action.d/firewallcmd-ipset.conf`: fixed create of set for ipv6 (missing `family inet6`, gh-1990)
-* `filter.d/sshd.conf`: extended failregex for modes "extra"/"aggressive": now finds all possible (also future)
-  forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", 
-  see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
+* `filter.d/sshd.conf`:
+  - extended failregex for modes "extra"/"aggressive": now finds all possible (also future)
+    forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", 
+    see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
+  - fixed failregex in order to avoid banning of legitimate users with multiple public keys (gh-2014, gh-1263);
 
 ### New Features
 * datedetector: extended default date-patterns (allows extra space between the date and time stamps);

From 29e1fe9479601ebc0cf80bc2ba4e625a312ed345 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Jan 2018 11:15:58 +0100
Subject: [PATCH 610/623] micro-fix: delete temporary file (forgotten in
 test-case `test_move_dir` by reassign to directory)

---
 fail2ban/tests/filtertestcase.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index d1567899..eb57c192 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -997,6 +997,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.file.close()
 			self.filter.setMaxRetry(10)
 			self.filter.delLogPath(self.name)
+			_killfile(None, self.name)
 			# if we rename parent dir into a new location (simulate directory-base log rotation)
 			tmpsub1 = os.path.join(tmp, "1")
 			tmpsub2 = os.path.join(tmp, "2")

From 7e05976ead14984a0c10d76f2e49713762479039 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Thu, 11 Jan 2018 12:38:34 +0100
Subject: [PATCH 611/623] action.d/hostsdeny.conf: actionunban rewritten using
 sed, also dots in IP were escaped now.

Closes  #2000
---
 config/action.d/hostsdeny.conf | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/config/action.d/hostsdeny.conf b/config/action.d/hostsdeny.conf
index d74f498d..5cca6529 100644
--- a/config/action.d/hostsdeny.conf
+++ b/config/action.d/hostsdeny.conf
@@ -31,8 +31,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = IP=<ip> &&
-            printf %%b "<daemon_list>: $IP\n" >> <file>
+actionban = IP=<ip> && printf %%b "<daemon_list>: $IP\n" >> <file>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -40,7 +39,7 @@ actionban = IP=<ip> &&
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = echo "/^<daemon_list>: <ip>$/<br>d<br>w<br>q" | ed <file>
+actionunban = IP=$(echo <ip> | sed 's/\./\\./g') && sed -i "/^<daemon_list>: $IP$/d" <file>
 
 [Init]
 

From 7516cd025d22dd82a700a2690455973bc33ca2aa Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Jan 2018 13:21:36 +0100
Subject: [PATCH 612/623] fixed restoring sane environment (via stop/start) if
 invariant check failed: bypass possible errors in stop (if start/check
 succeeded hereafter); test cases extended to cover such situation. Closes
 gh-1997

---
 fail2ban/server/action.py        |  5 ++++-
 fail2ban/tests/actiontestcase.py | 11 ++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 81ffbe2f..31475d72 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -750,7 +750,10 @@ class CommandAction(ActionBase):
 					#    otherwise stop/start will theoretically remove all the bans,
 					#    but the tickets are still in BanManager, so in case of new failures
 					#    it will not be banned, because "already banned" will happen.
-					self.stop()
+					try:
+						self.stop()
+					except RuntimeError: # bypass error in stop (if start/check succeeded hereafter).
+						pass
 					self.start()
 				if not self.executeCmd(checkCmd, self.timeout):
 					self._logSys.critical("Unable to restore environment")
diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py
index 4c92057b..d05d4562 100644
--- a/fail2ban/tests/actiontestcase.py
+++ b/fail2ban/tests/actiontestcase.py
@@ -335,7 +335,16 @@ class CommandActionTest(LogCaptureTestCase):
 		self.__action.actionban = "rm /tmp/fail2ban.test"
 		self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]"
 		self.assertRaises(RuntimeError, self.__action.ban, {'ip': None})
-		self.assertLogged('Unable to restore environment')
+		self.assertLogged('Invariant check failed', 'Unable to restore environment', all=True)
+		# 2nd time, try to restore with producing error in stop, but succeeded start hereafter:
+		self.pruneLog('[phase 2]')
+		self.__action.actionstart = "touch /tmp/fail2ban.test"
+		self.__action.actionstop = "rm /tmp/fail2ban.test"
+		self.__action.actionban = 'printf "%%b\n" <ip> >> /tmp/fail2ban.test'
+		self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]"
+		self.__action.ban({'ip': None})
+		self.assertLogged('Invariant check failed')
+		self.assertNotLogged('Unable to restore environment')
 
 	def testExecuteActionCheckRepairEnvironment(self):
 		self.__action.actionstart = ""

From b644d2d73ffa2d1e468b148004f9fb3436efd507 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 11 Jan 2018 20:23:22 +0100
Subject: [PATCH 613/623] should fix sporadic coverage decrease (don't cover
 "return", because too sporadic to get idle in pyinotify-callback);

---
 fail2ban/server/filterpyinotify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index e59c203e..d63d1de6 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -123,7 +123,7 @@ class FilterPyinotify(FileFilter):
 			self._addPending(path, event)
 			return
 		# do nothing if idle:
-		if self.idle:
+		if self.idle: # pragma: no cover (too sporadic to get idle in callback)
 			return
 		# be sure we process a file:
 		if not isWF:

From 81c86fa83f523ab82a8ea0cc229a3a32acb13105 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Jan 2018 17:12:07 +0100
Subject: [PATCH 614/623] Remove annoying error-message "rm_watch: cannot
 remove WD=2, Errno=Invalid argument (EINVAL)", logged from pyinotify-module
 if rm_watch called with non-existing watch file descriptor (probably
 multi-threaded issue by dual-remove). Closes gh-1865

---
 fail2ban/server/filterpyinotify.py | 34 +++++++++++++++---------------
 fail2ban/tests/filtertestcase.py   | 15 +++++++++++++
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index d63d1de6..ef26303a 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -225,22 +225,27 @@ class FilterPyinotify(FileFilter):
 		self.__watchFiles.update(wd)
 		logSys.debug("Added file watcher for %s", path)
 
+	def _delWatch(self, wdInt):
+		m = self.__monitor
+		try:
+			if m.get_path(wdInt) is not None:
+				wd = m.rm_watch(wdInt, quiet=False)
+				return True
+		except pyinotify.WatchManagerError as e:
+			if m.get_path(wdInt) is not None and not str(e).endswith("(EINVAL)"): # prama: no cover
+				logSys.debug("Remove watch causes: %s", e)
+				raise e
+		return False
+
 	def _delFileWatcher(self, path):
 		try:
 			wdInt = self.__watchFiles.pop(path)
-			if self.__monitor.get_path(wdInt) is not None:
-				wd = self.__monitor.rm_watch(wdInt)
-			else: # pragma: no cover
+			if not self._delWatch(wdInt): # pragma: no cover
 				logSys.debug("Non-existing file watcher %r for file %s", wdInt, path)
-				wd = {wdInt: 1}
-			if wd[wdInt]:
-				logSys.debug("Removed file watcher for %s", path)
-				return True
+			logSys.debug("Removed file watcher for %s", path)
+			return True
 		except KeyError: # pragma: no cover
 			pass
-			# EnvironmentError is parent of IOError, OSError, etc.
-		except EnvironmentError as e: # pragma: no cover (normally unreached)
-			logSys.error("Remove file monitor for %s causes %s", path, e)
 		return False
 
 	def _addDirWatcher(self, path_dir):
@@ -256,16 +261,11 @@ class FilterPyinotify(FileFilter):
 		# Remove watches for the directory:
 		try:
 			wdInt = self.__watchDirs.pop(path_dir)
-			if self.__monitor.get_path(wdInt) is not None:
-				self.__monitor.rm_watch(wdInt)
-			else: # pragma: no cover
+			if not self._delWatch(wdInt): # pragma: no cover
 				logSys.debug("Non-existing file watcher %r for directory %s", wdInt, path_dir)
+			logSys.debug("Removed monitor for the parent directory %s", path_dir)
 		except KeyError: # pragma: no cover
 			pass
-			# EnvironmentError is parent of IOError, OSError, etc.
-		except EnvironmentError as e: # pragma: no cover (normally unreached)
-			logSys.error("Remove monitor for the parent directory %s causes %s", path_dir, e)
-		logSys.debug("Removed monitor for the parent directory %s", path_dir)
 
 	##
 	# Add a log file path
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index eb57c192..1803974b 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -980,6 +980,21 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)
 
+		def test_pyinotify_delWatch(self):
+			if hasattr(self.filter, '_delWatch'): # pyinotify only
+				m = self.filter._FilterPyinotify__monitor
+				# remove existing watch:
+				self.assertTrue(self.filter._delWatch(m.get_wd(self.name)))
+				# mockup get_path to allow once find path for invalid wd-value:
+				_org_get_path = m.get_path
+				def _get_path(wd):
+					#m.get_path = _org_get_path
+					return 'test'
+				m.get_path = _get_path
+				# try remove watch using definitely not existing handle:
+				self.assertFalse(self.filter._delWatch(0x7fffffff))
+				m.get_path = _org_get_path
+
 		def test_del_file(self):
 			# test filter reaction by delete watching file:
 			self.file.close()

From 2bce0c5e3ee805779359b02e4ad5272b32553e77 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 15 Jan 2018 18:00:15 +0100
Subject: [PATCH 615/623] file-filter's: provide stop function in order to
 explicitly delete/stop monitoring of each file.

---
 fail2ban/server/filter.py          | 9 +++++++++
 fail2ban/server/filterpyinotify.py | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c309bbb2..25a0e687 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -1061,6 +1061,15 @@ class FileFilter(Filter):
 		ret.append(("File list", path))
 		return ret
 
+	def stop(self):
+		"""Stop monitoring of log-file(s)
+		"""
+		# stop files monitoring:
+		for path in self.__logs.keys():
+			self.delLogPath(path)
+		# stop thread:
+		super(Filter, self).stop()
+
 ##
 # FileContainer class.
 #
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index ef26303a..17d387a1 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -78,6 +78,7 @@ class FilterPyinotify(FileFilter):
 		self.__modified = False
 		# Pyinotify watch manager
 		self.__monitor = pyinotify.WatchManager()
+		self.__notifier = None
 		self.__watchFiles = dict()
 		self.__watchDirs = dict()
 		self.__pending = dict()
@@ -240,7 +241,7 @@ class FilterPyinotify(FileFilter):
 	def _delFileWatcher(self, path):
 		try:
 			wdInt = self.__watchFiles.pop(path)
-			if not self._delWatch(wdInt): # pragma: no cover
+			if not self._delWatch(wdInt):
 				logSys.debug("Non-existing file watcher %r for file %s", wdInt, path)
 			logSys.debug("Removed file watcher for %s", path)
 			return True

From 6a1bbbf1013da4d0d03eb2b282f44f6b0ed8fad2 Mon Sep 17 00:00:00 2001
From: riceru <35490247+riceru@users.noreply.github.com>
Date: Tue, 16 Jan 2018 12:39:55 +0000
Subject: [PATCH 616/623] Update lighttpd-auth.conf

I have lighttpd 1.4.45 (Debian 9) and auth error log is different.
Now printing mod_auth and not http_auth.
I think that the change was in Lighttp 1.4.42
---
 config/filter.d/lighttpd-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf
index 3bd01f29..11f6d0bb 100644
--- a/config/filter.d/lighttpd-auth.conf
+++ b/config/filter.d/lighttpd-auth.conf
@@ -3,7 +3,7 @@
 
 [Definition]
 
-failregex = ^: \(http_auth\.c\.\d+\) (password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
+failregex = ^: \((http|mod)_auth\.c\.\d+\) (password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
 
 ignoreregex = 
 

From 9a46590486f8f9fe4f14f785ddbdd448c2bb7a24 Mon Sep 17 00:00:00 2001
From: "Sergey G. Brester" <github@sebres.de>
Date: Tue, 16 Jan 2018 14:20:51 +0100
Subject: [PATCH 617/623] extended test-cases to cover new log-format
 (http_auth -> mod_auth)

---
 fail2ban/tests/files/logs/lighttpd-auth | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fail2ban/tests/files/logs/lighttpd-auth b/fail2ban/tests/files/logs/lighttpd-auth
index a373c652..184dba33 100644
--- a/fail2ban/tests/files/logs/lighttpd-auth
+++ b/fail2ban/tests/files/logs/lighttpd-auth
@@ -5,3 +5,5 @@
 2012-09-26 10:24:35: (http_auth.c.1136) digest: auth failed for  xxx : wrong password, IP: 4.4.4.4
 # failJSON: { "time": "2013-08-25T00:24:55", "match": true , "host": "4.4.4.4" }
 2013-08-25 00:24:55: (http_auth.c.877) get_password failed, IP: 4.4.4.4
+# failJSON: { "time": "2018-01-16T14:10:32", "match": true , "host": "192.0.2.1", "desc": "http_auth -> mod_auth, gh-2018" }
+2018-01-16 14:10:32: (mod_auth.c.525) password doesn't match for /test-url username: test, IP: 192.0.2.1

From b6c6565a7ed9229ab14070a9ed83d26655f7f9c0 Mon Sep 17 00:00:00 2001
From: "Sergey G. Brester" <github@sebres.de>
Date: Tue, 16 Jan 2018 14:23:47 +0100
Subject: [PATCH 618/623] regex updated using non-capturing groups

---
 config/filter.d/lighttpd-auth.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf
index 11f6d0bb..a68f4f4d 100644
--- a/config/filter.d/lighttpd-auth.conf
+++ b/config/filter.d/lighttpd-auth.conf
@@ -3,7 +3,7 @@
 
 [Definition]
 
-failregex = ^: \((http|mod)_auth\.c\.\d+\) (password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
+failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$
 
 ignoreregex = 
 

From fed6c49c2de2efd677788eeff0cae2e5aa4b9976 Mon Sep 17 00:00:00 2001
From: Benedikt Seidl <Benedikt.Seidl@gmx.de>
Date: Thu, 11 Jan 2018 18:53:15 +0100
Subject: [PATCH 619/623] nginx-http-auth: match usernames with spaces

# Conflicts:
#	ChangeLog
---
 ChangeLog                                 | 1 +
 config/filter.d/nginx-http-auth.conf      | 2 +-
 fail2ban/tests/files/logs/nginx-http-auth | 2 ++
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index ec11539f..d19356cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -40,6 +40,7 @@ releases.
       input SMTP command (lower/mixed case auth command, prevent injection) (gh-1979)
 * filter.d/postfix-*.conf - added optional port regex (gh-1902)
 * filter.d/sendmail-auth.conf - extended daemon for Fedora 24/RHEL - the daemon name is "sendmail" (gh-1632)
+* filter.d/nginx-http-auth.conf - match usernames with spaces (gh-2015)
 
 ### New Features
 
diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf
index a689f66a..511b5bf0 100644
--- a/config/filter.d/nginx-http-auth.conf
+++ b/config/filter.d/nginx-http-auth.conf
@@ -4,7 +4,7 @@
 [Definition]
 
 
-failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+failregex = ^ \[error\] \d+#\d+: \*\d+ user "[^"]+":? (password mismatch|was not found in ".*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth
index 22c16057..e54edce8 100644
--- a/fail2ban/tests/files/logs/nginx-http-auth
+++ b/fail2ban/tests/files/logs/nginx-http-auth
@@ -7,4 +7,6 @@
 2014/04/01 22:20:38 [error] 30708#0: *3 user "scribendio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
 # failJSON: { "time": "2014-04-02T12:37:58", "match": true, "host": "10.0.2.2" }
 2014/04/02 12:37:58 [error] 6563#0: *1861 user "scribendio": password mismatch, client: 10.0.2.2, server: scribend.io, request: "GET /admin HTTP/1.1", host: "scribend.io", referrer: "https://scribend.io/admin"
+# failJSON: { "time": "2014-04-01T22:20:38", "match": true, "host": "10.0.2.2" }
+2014/04/01 22:20:38 [error] 30708#0: *3 user "scriben dio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
 

From 63e906b2c1a889f6ef0f0739c909b2ab58bbf95b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 17 Jan 2018 16:31:43 +0100
Subject: [PATCH 620/623] regex rewritten: a bit fewer vulnerable now and using
 non-capturing groups, test-cases extended in order to cover trying of
 injection on user name

---
 config/filter.d/nginx-http-auth.conf      | 2 +-
 fail2ban/tests/files/logs/nginx-http-auth | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf
index 511b5bf0..65a8dd1b 100644
--- a/config/filter.d/nginx-http-auth.conf
+++ b/config/filter.d/nginx-http-auth.conf
@@ -4,7 +4,7 @@
 [Definition]
 
 
-failregex = ^ \[error\] \d+#\d+: \*\d+ user "[^"]+":? (password mismatch|was not found in ".*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+failregex = ^ \[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$
 
 ignoreregex = 
 
diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth
index e54edce8..c9c96807 100644
--- a/fail2ban/tests/files/logs/nginx-http-auth
+++ b/fail2ban/tests/files/logs/nginx-http-auth
@@ -7,6 +7,7 @@
 2014/04/01 22:20:38 [error] 30708#0: *3 user "scribendio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
 # failJSON: { "time": "2014-04-02T12:37:58", "match": true, "host": "10.0.2.2" }
 2014/04/02 12:37:58 [error] 6563#0: *1861 user "scribendio": password mismatch, client: 10.0.2.2, server: scribend.io, request: "GET /admin HTTP/1.1", host: "scribend.io", referrer: "https://scribend.io/admin"
-# failJSON: { "time": "2014-04-01T22:20:38", "match": true, "host": "10.0.2.2" }
-2014/04/01 22:20:38 [error] 30708#0: *3 user "scriben dio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
-
+# failJSON: { "time": "2014-04-03T22:20:38", "match": true, "host": "192.0.2.1", "desc": "user name with space" }
+2014/04/03 22:20:38 [error] 30708#0: *3 user "scriben dio": password mismatch, client: 192.0.2.1, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
+# failJSON: { "time": "2014-04-03T22:20:40", "match": true, "host": "192.0.2.2", "desc": "trying injection on user name"}
+2014/04/03 22:20:40 [error] 30708#0: *3 user "test": password mismatch, client: 127.0.0.1, server: test, request: "GET / HTTP/1.1", host: "localhost:8443"": was not found in "/etc/nginx/.htpasswd", client: 192.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"

From f69e28adfc754f74ca87a104031d949c201013c7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 18 Jan 2018 14:05:22 +0100
Subject: [PATCH 621/623] action.d/pf.conf: compatibility fix - recognizes that
 parameter `port` specified as empty, with or without braces (should be more
 backwards compatible to 0.9 now).

---
 config/action.d/pf.conf          | 5 +++--
 fail2ban/tests/servertestcase.py | 6 ++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/config/action.d/pf.conf b/config/action.d/pf.conf
index b3a33cf2..905312c3 100644
--- a/config/action.d/pf.conf
+++ b/config/action.d/pf.conf
@@ -26,7 +26,8 @@
 #     }
 # to your main pf ruleset, where "namei" are the names of the jails
 # which invoke this action
-actionstart = echo "table <<tablename>-<name>> persist counters" | <pfctl> -f- 
+actionstart = echo "table <<tablename>-<name>> persist counters" | <pfctl> -f-
+              port="<port>"; if [ "$port" != "" ] && case "$port" in \{*) false;; esac; then port="{$port}"; fi
               echo "<block> proto <protocol> from <<tablename>-<name>> to <actiontype>" | <pfctl> -f-
 
 # Option:  start_on_demand - to start action on demand
@@ -119,5 +120,5 @@ allports = any
 # Option: multiport
 # Notes.: addition to block access only to specific ports
 # Usage.: use in jail config: "banaction = pf[actiontype=<multiport>]"
-multiport = any port {<port>}
+multiport = any port $port
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index a5e93d2d..8485aa74 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -1503,7 +1503,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -a f2b/j-w-pf -f-`',
-						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port {<port>}" | pfctl -a f2b/j-w-pf -f-`',
+						'port="<port>"',
+						'`echo "block quick proto tcp from <f2b-j-w-pf> to any port $port" | pfctl -a f2b/j-w-pf -f-`',
 					),
 					'flush': (
 						'`pfctl -a f2b/j-w-pf -t f2b-j-w-pf -T flush`',
@@ -1525,7 +1526,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					'ip4': (), 'ip6': (),
 					'start': (
 						'`echo "table <f2b-j-w-pf-mp> persist counters" | pfctl -a f2b/j-w-pf-mp -f-`',
-						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port {http,https}" | pfctl -a f2b/j-w-pf-mp -f-`',
+						'port="http,https"',
+						'`echo "block quick proto tcp from <f2b-j-w-pf-mp> to any port $port" | pfctl -a f2b/j-w-pf-mp -f-`',
 					),
 					'flush': (
 						'`pfctl -a f2b/j-w-pf-mp -t f2b-j-w-pf-mp -T flush`',

From 81b61fe30c062332febc4966ed54cf4f1b226437 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 18 Jan 2018 14:19:55 +0100
Subject: [PATCH 622/623] ChangeLog update

---
 ChangeLog | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6052b0f1..a2fb8390 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,7 +15,7 @@ Incompatibility list (compared to v.0.9):
     rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)`
     (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings).
 
-    Of course you can always your own capture-group (like below `_cond_ip_`) to do this.
+    Of course you can always define your own capture-group (like below `_cond_ip_`) to do this.
     ```
     testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1"
     fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$"
@@ -35,8 +35,9 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition
 -----------
 
 ### Incompatibility list:
-* The configuration for multiport jails using banaction `pf` can be incompatible after upgrade, if
-  ports are enclosed in curly braces `{ }` in the `jail.local` etc. This may cause a double-brackets now.
+* The configuration for jails using banaction `pf` can be incompatible after upgrade, because pf-action uses
+  anchors now (see `action.d/pf.conf` for more information). If you want use obsolete handling without anchors,
+  just rewrite it in the `jail.local` by overwrite of `pfctl` parameter, e. g. like `banaction = pf[pfctl="pfctl"]`.
 
 ### Fixes
 * Fixed logging to systemd-journal: new logtarget value SYSOUT can be used instead of STDOUT, to avoid 

From a45488465e0dd547eb8479c0fa9fd577c1837213 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 18 Jan 2018 14:46:50 +0100
Subject: [PATCH 623/623] prepare release: bump version, update ChangeLog,
 man's and MANIFEST etc.

---
 ChangeLog                | 2 +-
 MANIFEST                 | 1 +
 README.md                | 6 +++---
 fail2ban/version.py      | 2 +-
 man/fail2ban-client.1    | 4 ++--
 man/fail2ban-regex.1     | 2 +-
 man/fail2ban-server.1    | 4 ++--
 man/fail2ban-testcases.1 | 2 +-
 8 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a2fb8390..29b07973 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -31,7 +31,7 @@ Incompatibility list (compared to v.0.9):
   IPv6-capable now.
 
 
-ver. 0.10.2-dev-1 (2017/??/??) - development edition
+ver. 0.10.2 (2018/01/18) - nothing-burns-like-the-cold
 -----------
 
 ### Incompatibility list:
diff --git a/MANIFEST b/MANIFEST
index d69ca029..bfdd77c0 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -44,6 +44,7 @@ config/action.d/netscaler.conf
 config/action.d/nftables-allports.conf
 config/action.d/nftables-common.conf
 config/action.d/nftables-multiport.conf
+config/action.d/nginx-block-map.conf
 config/action.d/npf.conf
 config/action.d/nsupdate.conf
 config/action.d/osx-afctl.conf
diff --git a/README.md b/README.md
index 0821e691..94485adb 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.2                 2017/??/??
+                       v0.10.2                 2018/01/18
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
@@ -43,8 +43,8 @@ Optional:
 
 To install, just do:
 
-    tar xvfj fail2ban-0.10.1.tar.bz2
-    cd fail2ban-0.10.1
+    tar xvfj fail2ban-0.10.2.tar.bz2
+    cd fail2ban-0.10.2
     python setup.py install
 
 This will install Fail2Ban into the python library directory. The executable
diff --git a/fail2ban/version.py b/fail2ban/version.py
index 2fa9b6a5..67f9a6e6 100644
--- a/fail2ban/version.py
+++ b/fail2ban/version.py
@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.10.2.dev1"
+version = "0.10.2"
diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1
index 6e44b387..ab9559fb 100644
--- a/man/fail2ban-client.1
+++ b/man/fail2ban-client.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "October 2017" "fail2ban-client v0.10.2.dev1" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "January 2018" "fail2ban-client v0.10.2" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.10.2.dev1 reads log file that contains password failure report
+Fail2Ban v0.10.2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1
index a2347b1f..c4fca08c 100644
--- a/man/fail2ban-regex.1
+++ b/man/fail2ban-regex.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "October 2017" "fail2ban-regex 0.10.2.dev1" "User Commands"
+.TH FAIL2BAN-REGEX "1" "January 2018" "fail2ban-regex 0.10.2" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS
diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1
index aca3eeb0..debfb232 100644
--- a/man/fail2ban-server.1
+++ b/man/fail2ban-server.1
@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "October 2017" "fail2ban-server v0.10.2.dev1" "User Commands"
+.TH FAIL2BAN-SERVER "1" "January 2018" "fail2ban-server v0.10.2" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.10.2.dev1 reads log file that contains password failure report
+Fail2Ban v0.10.2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP
diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1
index 6feb0162..5762f827 100644
--- a/man/fail2ban-testcases.1
+++ b/man/fail2ban-testcases.1
@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "October 2017" "fail2ban-testcases 0.10.2.dev1" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "January 2018" "fail2ban-testcases 0.10.2" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS