diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 29723dfb..87d359ba 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -694,6 +694,14 @@ def exec_command_line(*args): stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt))) logSys.addHandler(stdout) - fail2banRegex = Fail2banRegex(opts) + try: + fail2banRegex = Fail2banRegex(opts) + except Exception as e: + if opts.verbose or logSys.getEffectiveLevel()<=logging.DEBUG: + logSys.critical(e, exc_info=True) + else: + output( 'ERROR: %s' % e ) + sys.exit(255) + if not fail2banRegex.start(args): sys.exit(255) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e032c2b0..d827cd1a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -301,14 +301,17 @@ class DatePatternRegex(DateTemplate): if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern): pattern = RE_EXLINE_BOUND_BEG.sub('', pattern) wordBegin = 'start' - # wrap to regex: - fmt = self._patternRE.sub(r'%(\1)s', pattern) - self.name = fmt % self._patternName - regex = fmt % timeRE - # if expected add (?iu) for "ignore case" and "unicode": - if RE_ALPHA_PATTERN.search(pattern): - regex = r'(?iu)' + regex - super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) + try: + # wrap to regex: + fmt = self._patternRE.sub(r'%(\1)s', pattern) + self.name = fmt % self._patternName + regex = fmt % timeRE + # if expected add (?iu) for "ignore case" and "unicode": + if RE_ALPHA_PATTERN.search(pattern): + regex = r'(?iu)' + regex + super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) + except Exception as e: + raise TypeError("Failed to set datepattern '%s' (may be an invalid format or unescaped percent char): %s" % (pattern, e)) def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 959e39c4..7256ce4e 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -197,7 +197,7 @@ class IPAddr(object): __slots__ = '_family','_addr','_plen','_maskplen','_raw' # todo: make configurable the expired time and max count of cache entries: - CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60) + CACHE_OBJ = Utils.Cache(maxCount=10000, maxTime=5*60) CIDR_RAW = -2 CIDR_UNSPEC = -1 @@ -205,6 +205,10 @@ class IPAddr(object): FAM_IPv6 = CIDR_RAW - socket.AF_INET6 def __new__(cls, ipstr, cidr=CIDR_UNSPEC): + if cidr == IPAddr.CIDR_RAW: # don't cache raw + ip = super(IPAddr, cls).__new__(cls) + ip.__init(ipstr, cidr) + return ip # check already cached as IPAddr args = (ipstr, cidr) ip = IPAddr.CACHE_OBJ.get(args) @@ -221,7 +225,8 @@ class IPAddr(object): return ip ip = super(IPAddr, cls).__new__(cls) ip.__init(ipstr, cidr) - IPAddr.CACHE_OBJ.set(args, ip) + if ip._family != IPAddr.CIDR_RAW: + IPAddr.CACHE_OBJ.set(args, ip) return ip @staticmethod diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 73a376e0..4e61c420 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -95,31 +95,35 @@ class Utils(): def set(self, k, v): t = time.time() - cache = self._cache # for shorter local access - # clean cache if max count reached: - if len(cache) >= self.maxCount: - # avoid multiple modification of list multi-threaded: - with self.__lock: - if len(cache) >= self.maxCount: - for (ck, cv) in cache.items(): + # avoid multiple modification of dict multi-threaded: + cache = self._cache + with self.__lock: + # clean cache if max count reached: + if len(cache) >= self.maxCount: + if OrderedDict is not dict: + # ordered (so remove some from ahead, FIFO) + while cache: + (ck, cv) = cache.popitem(last=False) + # if not yet expired (but has free slot for new entry): + if cv[1] > t and len(cache) < self.maxCount: + break + else: # pragma: 3.x no cover (dict is in 2.6 only) + remlst = [] + for (ck, cv) in cache.iteritems(): # if expired: if cv[1] <= t: - self.unset(ck) - elif OrderedDict is not dict: - break + remlst.append(ck) + for ck in remlst: + self._cache.pop(ck, None) # if still max count - remove any one: - if len(cache) >= self.maxCount: - if OrderedDict is not dict: # first (older): - cache.popitem(False) - else: # pragma: 3.x no cover - cache.popitem() - cache[k] = (v, t + self.maxTime) + while cache and len(cache) >= self.maxCount: + cache.popitem() + # set now: + cache[k] = (v, t + self.maxTime) def unset(self, k): - try: - del self._cache[k] - except KeyError: - pass + with self.__lock: + self._cache.pop(k, None) @staticmethod diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 44acfd35..aa6977ed 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -368,3 +368,16 @@ class Fail2banRegexTest(LogCaptureTestCase): r"Authentication failure" ), 0) self.assertLogged('No failure-id group in ') + + def testExecCmdLine_ErrorParam(self): + # single line error: + self.assertNotEqual(_test_exec_command_line( + '-l', 'notice', '-d', '%:%.%-', 'LOG', 'RE' + ), 0) + self.assertLogged('ERROR: Failed to set datepattern') + # verbose (traceback/callstack): + self.pruneLog() + self.assertNotEqual(_test_exec_command_line( + '-v', '-d', '%:%.%-', 'LOG', 'RE' + ), 0) + self.assertLogged('Failed to set datepattern') diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index cae1e173..08d97b1f 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1753,6 +1753,44 @@ class DNSUtilsTests(unittest.TestCase): # here the whole cache should be empty: self.assertEqual(len(c), 0) + def testOverflowedIPCache(self): + # test overflow of IP-cache multi-threaded (2 "parasite" threads flooding cache): + from threading import Thread + from random import shuffle + # save original cache and use smaller cache during the test here: + _org_cache = IPAddr.CACHE_OBJ + cache = IPAddr.CACHE_OBJ = Utils.Cache(maxCount=5, maxTime=60) + result = list() + count = 1 if unittest.F2B.fast else 50 + try: + # tester procedure of worker: + def _TestCacheStr2IP(forw=True, result=[], random=False): + try: + c = count + while c: + c -= 1 + s = xrange(0, 256, 1) if forw else xrange(255, -1, -1) + if random: shuffle([i for i in s]) + for i in s: + IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4) + IPAddr('2001:db8::'+str(i), IPAddr.FAM_IPv6) + result.append(None) + except Exception as e: + DefLogSys.debug(e, exc_info=True) + result.append(e) + + # 2 workers flooding it forwards and backwards: + th1 = Thread(target=_TestCacheStr2IP, args=(True, result)); th1.start() + th2 = Thread(target=_TestCacheStr2IP, args=(False, result)); th2.start() + # and here we flooding it with random IPs too: + _TestCacheStr2IP(True, result, True) + finally: + # wait for end of threads and restore cache: + th1.join() + th2.join() + IPAddr.CACHE_OBJ = _org_cache + self.assertEqual(result, [None]*3) # no errors + self.assertTrue(len(cache) <= cache.maxCount) class DNSUtilsNetworkTests(unittest.TestCase):