From f8f01d5ab771d9eff0f8f6a391b28456f419bf05 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 9 Jul 2018 14:58:39 +0200
Subject: [PATCH] introduced new option `ignorecache` to improve performance of
 ignore failure check (using caching of `ignoreip`, `ignoreself` and
 `ignorecommand`)

---
 ChangeLog                        |  2 ++
 fail2ban/client/jailreader.py    |  1 +
 fail2ban/protocol.py             |  3 +-
 fail2ban/server/filter.py        | 54 ++++++++++++++++++++++++++------
 fail2ban/server/server.py        | 11 +++++--
 fail2ban/server/transmitter.py   |  6 ++++
 fail2ban/tests/filtertestcase.py | 46 ++++++++++++++++++++++++---
 fail2ban/tests/servertestcase.py |  9 +++++-
 man/jail.conf.5                  | 14 ++++++++-
 9 files changed, 127 insertions(+), 19 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 782c557d..f54b4a71 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -50,6 +50,8 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition
 * systemd: fixed type error on option `journalflags`: an integer is required (gh-2125);
 
 ### New Features
+* new option `ignorecache` to improve performance of ignore failure check (using caching of `ignoreip`, 
+  `ignoreself` and `ignorecommand`), see `man jail.conf` for syntax-example;
 * `ignorecommand` extended to use actions-similar replacement (capable to interpolate 
   all possible tags like `<ip-host>`, `<family>`, `<fid>`, `F-USER` etc.)
 
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index 72bf780d..b06ba72d 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -100,6 +100,7 @@ class JailReader(ConfigReader):
 				["string", "ignorecommand", None],
 				["bool",   "ignoreself", None],
 				["string", "ignoreip", None],
+				["string", "ignorecache", None],
 				["string", "filter", ""],
 				["string", "datepattern", None],
 				["string", "logtimezone", None],
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 3625ec01..b21ab848 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -84,6 +84,8 @@ protocol = [
 ["set <JAIL> ignoreself true|false", "allows the ignoring of own IP addresses"], 
 ["set <JAIL> addignoreip <IP>", "adds <IP> to the ignore list of <JAIL>"], 
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
+["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
+["set <JAIL> ignorecache <VALUE>", "sets ignorecache of <JAIL>"],
 ["set <JAIL> addlogpath <FILE> ['tail']", "adds <FILE> to the monitoring list of <JAIL>, optionally starting at the 'tail' of the file (default 'head')."], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
 ["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
@@ -91,7 +93,6 @@ protocol = [
 ["set <JAIL> deljournalmatch <MATCH>", "removes <MATCH> from the journal filter of <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
-["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
 ["set <JAIL> delignoreregex <INDEX>", "removes the regular expression at <INDEX> for ignoreregex"], 
 ["set <JAIL> findtime <TIME>", "sets the number of seconds <TIME> for which the filter will look back for <JAIL>"], 
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 00c1dc54..681e708c 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -81,6 +81,10 @@ class Filter(JailThread):
 		self.__ignoreSelf = True
 		## The ignore IP list.
 		self.__ignoreIpList = []
+		## External command
+		self.__ignoreCommand = False
+		## Cache for ignoreip:
+		self.__ignoreCache = None
 		## Size of line buffer
 		self.__lineBufferSize = 1
 		## Line buffer
@@ -90,8 +94,6 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## if set, treat log lines without explicit time zone to be in this time zone
 		self.__logtimezone = None
-		## External command
-		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
 		self.__encoding = PREFER_ENC
 		## Cache temporary holds failures info (used by multi-line for wrapping e. g. conn-id to host):
@@ -397,19 +399,34 @@ class Filter(JailThread):
 		raise Exception("run() is abstract")
 
 	##
-	# Set external command, for ignoredips
+	# External command, for ignoredips
 	#
 
-	def setIgnoreCommand(self, command):
+	@property
+	def ignoreCommand(self):
+		return self.__ignoreCommand
+
+	@ignoreCommand.setter
+	def ignoreCommand(self, command):
 		self.__ignoreCommand = command
 
 	##
-	# Get external command, for ignoredips
+	# Cache parameters for ignoredips
 	#
 
-	def getIgnoreCommand(self):
-		return self.__ignoreCommand
+	@property
+	def ignoreCache(self):
+		return [self.__ignoreCache[0], self.__ignoreCache[1].maxCount, self.__ignoreCache[1].maxTime] \
+			if self.__ignoreCache else None
 
+	@ignoreCache.setter
+	def ignoreCache(self, command):
+		if command:
+			self.__ignoreCache = command['key'], Utils.Cache(
+				maxCount=int(command.get('max-count', 100)), maxTime=MyTime.str2seconds(command.get('max-time', 5*60))
+			)
+		else:
+			self.__ignoreCache = None
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
@@ -502,29 +519,48 @@ class Filter(JailThread):
 		return self._inIgnoreIPList(ip, ticket, log_ignore)
 
 	def _inIgnoreIPList(self, ip, ticket, log_ignore=True):
+		aInfo = None
+		# cached ?
+		if self.__ignoreCache:
+			key, c = self.__ignoreCache
+			if ticket:
+				aInfo = Actions.ActionInfo(ticket, self.jail)
+				key = CommandAction.replaceDynamicTags(key, aInfo)
+			else:
+				aInfo = { 'ip': ip }
+				key = CommandAction.replaceTag(key, aInfo)
+			v = c.get(key)
+			if v is not None:
+				return v
+
 		# check own IPs should be ignored and 'ip' is self IP:
 		if self.__ignoreSelf and ip in DNSUtils.getSelfIPs():
 			self.logIgnoreIp(ip, log_ignore, ignore_source="ignoreself rule")
+			if self.__ignoreCache: c.set(key, True)
 			return True
 
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
+				if self.__ignoreCache: c.set(key, True)
 				return True
 
 		if self.__ignoreCommand:
 			if ticket:
-				aInfo = Actions.ActionInfo(ticket, self.jail)
+				if not aInfo: aInfo = Actions.ActionInfo(ticket, self.jail)
 				command = CommandAction.replaceDynamicTags(self.__ignoreCommand, aInfo)
 			else:
-				command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip })
+				if not aInfo: aInfo = { 'ip': ip }
+				command = CommandAction.replaceTag(self.__ignoreCommand, aInfo)
 			logSys.debug('ignore command: %s', command)
 			ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1))
 			ret_ignore = ret and ret_ignore == 0
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
+			if self.__ignoreCache: c.set(key, ret_ignore)
 			return ret_ignore
 
+		if self.__ignoreCache: c.set(key, False)
 		return False
 
 	def processLine(self, line, date=None):
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 5ae3d807..55689a2b 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -391,10 +391,17 @@ class Server:
 		return self.__jails[name].filter.getLogTimeZone()
 
 	def setIgnoreCommand(self, name, value):
-		self.__jails[name].filter.setIgnoreCommand(value)
+		self.__jails[name].filter.ignoreCommand = value
 
 	def getIgnoreCommand(self, name):
-		return self.__jails[name].filter.getIgnoreCommand()
+		return self.__jails[name].filter.ignoreCommand
+
+	def setIgnoreCache(self, name, value):
+		value, options = extractOptions("cache["+value+"]")
+		self.__jails[name].filter.ignoreCache = options
+
+	def getIgnoreCache(self, name):
+		return self.__jails[name].filter.ignoreCache
 
 	def setPrefRegex(self, name, value):
 		flt = self.__jails[name].filter
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 1aa7f3e2..89879d47 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -200,6 +200,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setIgnoreCommand(name, value)
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			value = command[2]
+			self.__server.setIgnoreCache(name, value)
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "addlogpath":
 			value = command[2]
 			tail = False
@@ -358,6 +362,8 @@ class Transmitter:
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "ignorecommand":
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "prefregex":
 			return self.__server.getPrefRegex(name)
 		elif command[1] == "failregex":
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 3ea61b26..350d46b0 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -401,7 +401,7 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')
 
 	def testIgnoreCommand(self):
-		self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>"))
+		self.filter.ignoreCommand = sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>")
 		self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
 		self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
 		self.assertLogged("returned successfully 0", "returned successfully 1", all=True)
@@ -411,7 +411,7 @@ class IgnoreIP(LogCaptureTestCase):
 	
 	def testIgnoreCommandForTicket(self):
 		# by host of IP (2001:db8::1 and 2001:db8::ffff map to "test-host" and "test-other" in the test-suite):
-		self.filter.setIgnoreCommand('if [ "<ip-host>" = "test-host" ]; then exit 0; fi; exit 1')
+		self.filter.ignoreCommand = 'if [ "<ip-host>" = "test-host" ]; then exit 0; fi; exit 1'
 		self.pruneLog()
 		self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1")))
 		self.assertLogged("returned successfully 0")
@@ -419,7 +419,7 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff")))
 		self.assertLogged("returned successfully 1")
 		# by user-name (ignore tester):
-		self.filter.setIgnoreCommand('if [ "<F-USER>" = "tester" ]; then exit 0; fi; exit 1')
+		self.filter.ignoreCommand = 'if [ "<F-USER>" = "tester" ]; then exit 0; fi; exit 1'
 		self.pruneLog()
 		self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'})))
 		self.assertLogged("returned successfully 0")
@@ -427,6 +427,42 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'})))
 		self.assertLogged("returned successfully 1", all=True)
 
+	def testIgnoreCache(self):
+		# like both test-cases above, just cached (so once per key)...
+		self.filter.ignoreCache = {"key":"<ip>"}
+		self.filter.ignoreCommand = 'if [ "<ip>" = "10.0.0.1" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
+			self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
+			if not i:
+				self.assertLogged("returned successfully 0", "returned successfully 1", all=True)
+			else:
+				self.assertNotLogged("returned successfully 0", "returned successfully 1", all=True)
+		# by host of IP:
+		self.filter.ignoreCache = {"key":"<ip-host>"}
+		self.filter.ignoreCommand = 'if [ "<ip-host>" = "test-host" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1")))
+			self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff")))
+			if not i:
+				self.assertLogged("returned successfully")
+			else:
+				self.assertNotLogged("returned successfully")
+		# by user-name:
+		self.filter.ignoreCache = {"key":"<F-USER>", "max-count":"10", "max-time":"1h"}
+		self.assertEqual(self.filter.ignoreCache, ["<F-USER>", 10, 60*60])
+		self.filter.ignoreCommand = 'if [ "<F-USER>" = "tester" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'})))
+			self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'})))
+			if not i:
+				self.assertLogged("returned successfully")
+			else:
+				self.assertNotLogged("returned successfully")
+
 	def testIgnoreCauseOK(self):
 		ip = "93.184.216.34"
 		for ignore_source in ["dns", "ip", "command"]:
@@ -490,7 +526,7 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd])))
 		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"])))
 		## via command:
-		self.filter.setIgnoreCommand(cmd + " <ip>")
+		self.filter.ignoreCommand = cmd + " <ip>"
 		for ip in bot_ips:
 			self.assertTrue(self.filter.inIgnoreIPList(str(ip)), "test of googlebot ip %s failed" % ip)
 			self.assertLogged('-- returned successfully')
@@ -498,7 +534,7 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
 		self.assertLogged('Argument must be a single valid IP.')
 		self.pruneLog()
-		self.filter.setIgnoreCommand(cmd + " bad arguments <ip>")
+		self.filter.ignoreCommand = cmd + " bad arguments <ip>"
 		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
 		self.assertLogged('Please provide a single IP as an argument.')
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 5d52fa09..425ab757 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -463,7 +463,14 @@ class Transmitter(TransmitterBase):
 			(0, False))
 
 	def testJailIgnoreCommand(self):
-		self.setGetTest("ignorecommand", "bin ", jail=self.jailName)
+		self.setGetTest("ignorecommand", "bin/ignore-command <ip>", jail=self.jailName)
+
+	def testJailIgnoreCache(self):
+		self.setGetTest("ignorecache", 
+			'key="<ip>",max-time=1d,max-count=9999', 
+			["<ip>", 9999, 24*60*60],
+			jail=self.jailName)
+		self.setGetTest("ignorecache", '', None, jail=self.jailName)
 
 	def testJailRegex(self):
 		self.jailAddDelRegexTest("failregex",
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index e939771b..25b3dd70 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -233,7 +233,19 @@ list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option a
 command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit.
 .br
 IP will not be banned if command returns successfully (exit code 0).
-Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution. Currently only <ip> is supported however more will be added later.
+Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution.
+.TP
+.B ignorecache
+provide cache parameters (default disabled) for ignore failure check (caching of the result from `ignoreip`, `ignoreself` and `ignorecommand`), syntax:
+
+.RS
+.nf
+        ignorecache = key="<F-USER>@<ip-host>", max-count=100, max-time=5m
+        ignorecommand = if [ "<F-USER>" = "technical" ] && [ "<ip-host>" = "my-host.example.com" ]; then exit 0; fi;
+                        exit 1
+.fi
+This will cache the result of \fBignorecommand\fR (does not call it repeatedly) for 5 minutes (cache time) for maximal 100 entries (cache size), using values substituted like "user@host" as cache-keys.  Set option \fBignorecache\fR to empty value disables the cache.
+.RE
 .TP
 .B bantime
 effective ban duration (in seconds or time abbreviation format).