From ba44ff312b45502647c7ab367bcdff0a6609ac92 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 9 Sep 2014 14:55:34 +0800 Subject: [PATCH 01/31] grep IP at the start of lines I'm not sure if this regex works best, so I'm patching this single file as a sample. Don't forget to update `mail-whois-lines.conf` after this patch got merged. For the following logs, `grep '[^0-9]199.48.161.87[^0-9]'` will output nothing, while `grep '\([^0-9]\|^\)199.48.161.87[^0-9]'` works: 
199.48.161.87 - - [09/Sep/2014:13:38:54 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:38:56 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:38:58 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:00 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:05 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:05 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:13 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:21 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:32 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:34 +0800] "POST /wp-login.php HTTP/1.1" 403 4674 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:34 +0800] "POST /wp-login.php HTTP/1.1" 403 168 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:34 +0800] "POST /wp-login.php HTTP/1.1" 403 168 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:35 +0800] "POST /wp-login.php HTTP/1.1" 403 168 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:35 +0800] "POST /wp-login.php HTTP/1.1" 403 168 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
199.48.161.87 - - [09/Sep/2014:13:39:35 +0800] "POST /wp-login.php HTTP/1.1" 403 168 "-" "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0" - hitsjapan.com
--- config/action.d/sendmail-whois-lines.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 270373e7..985d0048 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -26,7 +26,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n Lines containing IP: in \n - `grep '[^0-9][^0-9]' `\n\n + `grep '\([^0-9]\|^\)[^0-9]' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From 7acddcbe4afd1df547da151414cbe4f245d60ca1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 23:45:51 -0400 Subject: [PATCH 02/31] Post-release boost to .dev --- ChangeLog | 12 +++++++++++- README.md | 2 +- RELEASE | 14 +++++++------- fail2ban/version.py | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index cc3999ce..ffddd52b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,19 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.9.1) 2014/10/29 +Fail2Ban (version 0.9.1.dev) 2014/10/29 ================================================================================ +ver. 0.9.2 (2014/XX/XXX) - wanna-be-released +----------- + +- Fixes: + +- New Features: + +- Enhancements: + + ver. 0.9.1 (2014/10/29) - better, faster, stronger ---------- diff --git a/README.md b/README.md index 6afb5457..8210837e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.1 2014/10/29 + v0.9.1.dev 2014/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/RELEASE b/RELEASE index 23bb8382..db425a45 100644 --- a/RELEASE +++ b/RELEASE @@ -61,24 +61,24 @@ Preparation * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: - tar -C /tmp -jxf dist/fail2ban-0.9.1.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.9.2.tar.bz2 * clean up current direcory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.1/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.2/ * Only differences should be files that you don't want distributed. * Ensure the tests work from the tarball:: - cd /tmp/fail2ban-0.9.1/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases + cd /tmp/fail2ban-0.9.2/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases * Add/finalize the corresponding entry in the ChangeLog * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.1.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.2.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -101,7 +101,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.1 + git tag -s 0.9.2 Pre Release =========== @@ -144,7 +144,7 @@ Pre Release * https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban - * An potentially to the fail2ban-users email list. + * And potentially to the fail2ban-users email list. * Wait for feedback from distributors @@ -185,7 +185,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.2 (2014/XX/XXX) - wanna-be-released + ver. 0.9.3 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: diff --git a/fail2ban/version.py b/fail2ban/version.py index f7b776cd..146b76aa 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2014 Yaroslav Halchenko, 2013-2013 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.1" +version = "0.9.1.dev" From 518cc92ccc0866c1b82bedbf8b48f9a603b5c737 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Sep 2014 19:57:55 +0200 Subject: [PATCH 03/31] actions: bug fix in lambdas in checkBan, because getBansMerged could return None (purge resp. asynchronous addBan), make the logic all around more stable; test cases: extended with test to check action together with database functionality (ex.: to verify lambdas in checkBan); database: getBansMerged should work within lock, using reentrant lock (cause call of getBans inside of getBansMerged); --- fail2ban/server/actions.py | 51 ++++++++++--- fail2ban/server/database.py | 71 ++++++++++--------- fail2ban/tests/databasetestcase.py | 22 +++++- .../tests/files/action.d/action_checkainfo.py | 14 ++++ 4 files changed, 116 insertions(+), 42 deletions(-) create mode 100644 fail2ban/tests/files/action.d/action_checkainfo.py diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c8e9c5d9..a36802bc 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -243,6 +243,44 @@ class Actions(JailThread, Mapping): logSys.debug(self._jail.name + ": action terminated") return True + def __getBansMerged(self, mi, idx): + """Helper for lamda to get bans merged once + + This function never returns None for ainfo lambdas - always a ticket (merged or single one) + and prevents any errors through merging (to guarantee ban actions will be executed). + [TODO] move merging to observer - here we could wait for merge and read already merged info from a database + + Parameters + ---------- + mi : dict + initial for lambda should contains {ip, ticket} + idx : str + key to get a merged bans : + 'all' - bans merged for all jails + 'jail' - bans merged for current jail only + + Returns + ------- + BanTicket + merged or self ticket only + """ + if idx in mi: + return mi[idx] if mi[idx] is not None else mi['ticket'] + try: + jail=self._jail + ip=mi['ip'] + mi[idx] = None + if idx == 'all': + mi[idx] = jail.database.getBansMerged(ip=ip) + elif idx == 'jail': + mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail) + except Exception as e: + logSys.error( + "Failed to get %s bans merged, jail '%s': %s", + idx, jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + return mi[idx] if mi[idx] is not None else mi['ticket'] + def __checkBan(self): """Check for IP address to ban. @@ -264,14 +302,11 @@ class Actions(JailThread, Mapping): aInfo["time"] = bTicket.getTime() aInfo["matches"] = "\n".join(bTicket.getMatches()) if self._jail.database is not None: - aInfo["ipmatches"] = lambda jail=self._jail: "\n".join( - jail.database.getBansMerged(ip=ip).getMatches()) - aInfo["ipjailmatches"] = lambda jail=self._jail: "\n".join( - jail.database.getBansMerged(ip=ip, jail=jail).getMatches()) - aInfo["ipfailures"] = lambda jail=self._jail: \ - jail.database.getBansMerged(ip=ip).getAttempt() - aInfo["ipjailfailures"] = lambda jail=self._jail: \ - jail.database.getBansMerged(ip=ip, jail=jail).getAttempt() + mi4ip = lambda idx, self=self, mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, idx) + aInfo["ipmatches"] = lambda: "\n".join(mi4ip('all').getMatches()) + aInfo["ipjailmatches"] = lambda: "\n".join(mi4ip('jail').getMatches()) + aInfo["ipfailures"] = lambda: mi4ip('all').getAttempt() + aInfo["ipjailfailures"] = lambda: mi4ip('jail').getAttempt() if self.__banManager.addBanTicket(bTicket): logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) for name, action in self._actions.iteritems(): diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 351d1829..bbf7adad 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -27,7 +27,7 @@ import sqlite3 import json import locale from functools import wraps -from threading import Lock +from threading import RLock from .mytime import MyTime from .ticket import FailTicket @@ -123,7 +123,7 @@ class Fail2BanDb(object): def __init__(self, filename, purgeAge=24*60*60): try: - self._lock = Lock() + self._lock = RLock() self._db = sqlite3.connect( filename, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES) @@ -365,6 +365,10 @@ class Fail2BanDb(object): del self._bansMergedCache[(ticket.getIP(), jail)] except KeyError: pass + try: + del self._bansMergedCache[(ticket.getIP(), None)] + except KeyError: + pass #TODO: Implement data parts once arbitrary match keys completed cur.execute( "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", @@ -455,40 +459,41 @@ class Fail2BanDb(object): in a list. When `ip` argument passed, a single `Ticket` is returned. """ - cacheKey = None - if bantime is None or bantime < 0: - cacheKey = (ip, jail) - if cacheKey in self._bansMergedCache: - return self._bansMergedCache[cacheKey] + with self._lock: + cacheKey = None + if bantime is None or bantime < 0: + cacheKey = (ip, jail) + if cacheKey in self._bansMergedCache: + return self._bansMergedCache[cacheKey] - tickets = [] - ticket = None + tickets = [] + ticket = None - results = list(self._getBans(ip=ip, jail=jail, bantime=bantime)) - if results: - prev_banip = results[0][0] - matches = [] - failures = 0 - for banip, timeofban, data in results: - #TODO: Implement data parts once arbitrary match keys completed - if banip != prev_banip: - ticket = FailTicket(prev_banip, prev_timeofban, matches) - ticket.setAttempt(failures) - tickets.append(ticket) - # Reset variables - prev_banip = banip - matches = [] - failures = 0 - matches.extend(data['matches']) - failures += data['failures'] - prev_timeofban = timeofban - ticket = FailTicket(banip, prev_timeofban, matches) - ticket.setAttempt(failures) - tickets.append(ticket) + results = list(self._getBans(ip=ip, jail=jail, bantime=bantime)) + if results: + prev_banip = results[0][0] + matches = [] + failures = 0 + for banip, timeofban, data in results: + #TODO: Implement data parts once arbitrary match keys completed + if banip != prev_banip: + ticket = FailTicket(prev_banip, prev_timeofban, matches) + ticket.setAttempt(failures) + tickets.append(ticket) + # Reset variables + prev_banip = banip + matches = [] + failures = 0 + matches.extend(data['matches']) + failures += data['failures'] + prev_timeofban = timeofban + ticket = FailTicket(banip, prev_timeofban, matches) + ticket.setAttempt(failures) + tickets.append(ticket) - if cacheKey: - self._bansMergedCache[cacheKey] = tickets if ip is None else ticket - return tickets if ip is None else ticket + if cacheKey: + self._bansMergedCache[cacheKey] = tickets if ip is None else ticket + return tickets if ip is None else ticket @commitandrollback def purge(self, cur): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index f0757e5b..aef7443f 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -32,18 +32,21 @@ import shutil from ..server.filter import FileContainer from ..server.mytime import MyTime from ..server.ticket import FailTicket +from ..server.actions import Actions from .dummyjail import DummyJail try: from ..server.database import Fail2BanDb except ImportError: Fail2BanDb = None +from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") -class DatabaseTest(unittest.TestCase): +class DatabaseTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" + super(DatabaseTest, self).setUp() if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover raise unittest.SkipTest( "Unable to import fail2ban database module as sqlite is not " @@ -55,6 +58,7 @@ class DatabaseTest(unittest.TestCase): def tearDown(self): """Call after every test case.""" + super(DatabaseTest, self).tearDown() if Fail2BanDb is None: # pragma: no cover return # Cleanup @@ -267,6 +271,22 @@ class DatabaseTest(unittest.TestCase): tickets = self.db.getBansMerged(bantime=-1) self.assertEqual(len(tickets), 2) + def testActionWithDB(self): + # test action together with database functionality + self.testAddJail() # Jail required + self.jail.database = self.db; + actions = Actions(self.jail) + actions.add( + "action_checkainfo", + os.path.join(TEST_FILES_DIR, "action.d/action_checkainfo.py"), + {}) + ticket = FailTicket("1.2.3.4", MyTime.time(), ['test', 'test']) + ticket.setAttempt(5) + self.jail.putFailTicket(ticket) + actions._Actions__checkBan() + self.assertTrue(self._is_logged("ban ainfo %s, %s, %s, %s" % (True, True, True, True))) + + def testPurge(self): if Fail2BanDb is None: # pragma: no cover return diff --git a/fail2ban/tests/files/action.d/action_checkainfo.py b/fail2ban/tests/files/action.d/action_checkainfo.py new file mode 100644 index 00000000..eec9cc85 --- /dev/null +++ b/fail2ban/tests/files/action.d/action_checkainfo.py @@ -0,0 +1,14 @@ + +from fail2ban.server.action import ActionBase + +class TestAction(ActionBase): + + def ban(self, aInfo): + self._logSys.info("ban ainfo %s, %s, %s, %s", + aInfo["ipmatches"] != '', aInfo["ipjailmatches"] != '', aInfo["ipfailures"] > 0, aInfo["ipjailfailures"] > 0 + ) + + def unban(self, aInfo): + pass + +Action = TestAction From 8dbc04aa06e0cbcb951f489393e81638c150e577 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Oct 2014 13:30:24 +0100 Subject: [PATCH 04/31] Test cases fixed: testFail2BanExceptHook - use local sys.__excepthook__ to check was really executed and prevent write error in stderr. --- fail2ban/tests/servertestcase.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 062284f7..362f4253 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -804,7 +804,7 @@ class RegexTests(unittest.TestCase): class _BadThread(JailThread): def run(self): - int("ignore this exception -- raised for testing") + raise RuntimeError('run bad thread exception') class LoggingTests(LogCaptureTestCase): @@ -814,7 +814,13 @@ class LoggingTests(LogCaptureTestCase): self.assertEqual(testLogSys.name, "fail2ban.name") def testFail2BanExceptHook(self): + prev_exchook = sys.__excepthook__ + x = [] + sys.__excepthook__ = lambda *args: x.append(args) badThread = _BadThread() badThread.start() badThread.join() self.assertTrue(self._is_logged("Unhandled exception")) + sys.__excepthook__ = prev_exchook + self.assertEqual(len(x), 1) + self.assertEqual(x[0][0], RuntimeError) From 36abb5ed969d537bbdb5ab4959826f1ef23c2ae1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 29 Oct 2014 13:08:51 -0400 Subject: [PATCH 05/31] BF: fix $ for % in jail.conf. Debian bug #767255 --- ChangeLog | 1 + config/jail.conf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ffddd52b..99480d57 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: + * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 - New Features: diff --git a/config/jail.conf b/config/jail.conf index d31e70d7..d119d229 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -286,7 +286,7 @@ maxretry = 2 [apache-shellshock] port = http,https -logpath = $(apache_error_log)s +logpath = %(apache_error_log)s maxretry = 1 [nginx-http-auth] @@ -723,4 +723,4 @@ port = 2222 [portsentry] enabled = false logpath = /var/lib/portsentry/portsentry.history -maxretry = 1 \ No newline at end of file +maxretry = 1 From 46a8899f20b6513036bb62faf5d0eb7b02b0ad00 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Oct 2014 19:27:45 +0100 Subject: [PATCH 06/31] code review --- fail2ban/server/actions.py | 4 +++- fail2ban/tests/servertestcase.py | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a36802bc..d145f338 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -244,7 +244,7 @@ class Actions(JailThread, Mapping): return True def __getBansMerged(self, mi, idx): - """Helper for lamda to get bans merged once + """Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside. This function never returns None for ainfo lambdas - always a ticket (merged or single one) and prevents any errors through merging (to guarantee ban actions will be executed). @@ -274,6 +274,8 @@ class Actions(JailThread, Mapping): mi[idx] = jail.database.getBansMerged(ip=ip) elif idx == 'jail': mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail) + else: # pragma: no cover + raise ValueError("Unknown value %r for idx" % (idx,)) except Exception as e: logSys.error( "Failed to get %s bans merged, jail '%s': %s", diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 362f4253..e018ed2f 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -817,10 +817,12 @@ class LoggingTests(LogCaptureTestCase): prev_exchook = sys.__excepthook__ x = [] sys.__excepthook__ = lambda *args: x.append(args) - badThread = _BadThread() - badThread.start() - badThread.join() - self.assertTrue(self._is_logged("Unhandled exception")) - sys.__excepthook__ = prev_exchook + try: + badThread = _BadThread() + badThread.start() + badThread.join() + self.assertTrue(self._is_logged("Unhandled exception")) + finally: + sys.__excepthook__ = prev_exchook self.assertEqual(len(x), 1) self.assertEqual(x[0][0], RuntimeError) From 92ba5ae09c54aebd0a244fab634e301045610628 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 29 Oct 2014 22:08:44 +0100 Subject: [PATCH 07/31] few confusing merge info helper --- fail2ban/server/actions.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index d145f338..a212aaf1 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -243,7 +243,7 @@ class Actions(JailThread, Mapping): logSys.debug(self._jail.name + ": action terminated") return True - def __getBansMerged(self, mi, idx): + def __getBansMerged(self, mi, overalljails=False): """Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside. This function never returns None for ainfo lambdas - always a ticket (merged or single one) @@ -253,29 +253,28 @@ class Actions(JailThread, Mapping): Parameters ---------- mi : dict - initial for lambda should contains {ip, ticket} - idx : str - key to get a merged bans : - 'all' - bans merged for all jails - 'jail' - bans merged for current jail only + merge info, initial for lambda should contains {ip, ticket} + overalljails : bool + switch to get a merged bans : + False - (default) bans merged for current jail only + True - bans merged for all jails of current ip address Returns ------- BanTicket merged or self ticket only """ + idx = 'all' if overalljails else 'jail' if idx in mi: return mi[idx] if mi[idx] is not None else mi['ticket'] try: jail=self._jail ip=mi['ip'] mi[idx] = None - if idx == 'all': + if overalljails: mi[idx] = jail.database.getBansMerged(ip=ip) - elif idx == 'jail': + else: mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail) - else: # pragma: no cover - raise ValueError("Unknown value %r for idx" % (idx,)) except Exception as e: logSys.error( "Failed to get %s bans merged, jail '%s': %s", @@ -304,11 +303,12 @@ class Actions(JailThread, Mapping): aInfo["time"] = bTicket.getTime() aInfo["matches"] = "\n".join(bTicket.getMatches()) if self._jail.database is not None: - mi4ip = lambda idx, self=self, mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, idx) - aInfo["ipmatches"] = lambda: "\n".join(mi4ip('all').getMatches()) - aInfo["ipjailmatches"] = lambda: "\n".join(mi4ip('jail').getMatches()) - aInfo["ipfailures"] = lambda: mi4ip('all').getAttempt() - aInfo["ipjailfailures"] = lambda: mi4ip('jail').getAttempt() + mi4ip = lambda overalljails=False, self=self, \ + mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails) + aInfo["ipmatches"] = lambda: "\n".join(mi4ip(True).getMatches()) + aInfo["ipjailmatches"] = lambda: "\n".join(mi4ip().getMatches()) + aInfo["ipfailures"] = lambda: mi4ip(True).getAttempt() + aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt() if self.__banManager.addBanTicket(bTicket): logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) for name, action in self._actions.iteritems(): From 01b2673e3406b770882b4c246e8f44a5582f4d6f Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 29 Oct 2014 16:27:37 -0600 Subject: [PATCH 08/31] Use multiport for firewallcmd-new --- config/action.d/firewallcmd-new.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index 9754e3f3..ac72a68a 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -10,9 +10,9 @@ before = iptables-common.conf actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b- firewall-cmd --direct --add-rule ipv4 filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- + firewall-cmd --direct --add-rule ipv4 filter 0 -m state --state NEW -p -m multiport --dports -j f2b- -actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p --dport -j f2b- +actionstop = firewall-cmd --direct --remove-rule ipv4 filter 0 -m state --state NEW -p -m multiport --dports -j f2b- firewall-cmd --direct --remove-rules ipv4 filter f2b- firewall-cmd --direct --remove-chain ipv4 filter f2b- @@ -43,7 +43,7 @@ chain = INPUT_direct # success # $ firewall-cmd --direct --add-rule ipv4 filter fail2ban-name 1000 -j RETURN # success -# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp --dport 22 -j fail2ban-name +# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 22 -j fail2ban-name # success # $ firewall-cmd --direct --get-chains ipv4 filter # fail2ban-name From 967485c2d00d9aff91aea535d2909deab0ed1ba4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 29 Oct 2014 23:14:47 -0400 Subject: [PATCH 09/31] improving grepping --- ChangeLog | 2 ++ config/action.d/mail-whois-lines.conf | 2 +- config/action.d/sendmail-whois-lines.conf | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 99480d57..0e107735 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Fixes: * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 + * grep'ing for IP in *mail-whois-lines.conf should now match also + at the begginning and EOL. Thanks Dean Lee - New Features: diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index aa7d0950..5f760ac8 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -42,7 +42,7 @@ actionban = printf %%b "Hi,\n Here is more information about :\n `whois || echo missing whois program`\n\n Lines containing IP: in \n - `grep '[^0-9][^0-9]' `\n\n + `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 985d0048..4169e82a 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -26,7 +26,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n Lines containing IP: in \n - `grep '\([^0-9]\|^\)[^0-9]' `\n\n + `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From 21be98362070210fc12660370eafe2dd2e7f0b63 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Thu, 30 Oct 2014 16:11:34 -0600 Subject: [PATCH 10/31] ChangeLog for firewallcmd-new multiport support --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 99480d57..d2226201 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: - Enhancements: + * Enable multiport for firewallcmd-new action. Closes gh-834 ver. 0.9.1 (2014/10/29) - better, faster, stronger From 6dfddbcdf62553d56483d4990e84261845f46508 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 7 Nov 2014 01:21:38 +0100 Subject: [PATCH 11/31] Bug fix in formatJournalEntry, gh-851 Unhandled exception in fail2ban 0.9.1 #851 --- fail2ban/server/filtersystemd.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index ce89d65a..3a42f61c 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -167,9 +167,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements.append(logentry['_HOSTNAME']) if logentry.get('SYSLOG_IDENTIFIER'): logelements.append(logentry['SYSLOG_IDENTIFIER']) - if logentry.get('SYSLOG_PID') or logentry.get('_PID'): - logelements[-1] += ("[%i]" % logentry.get( - 'SYSLOG_PID', logentry['_PID'])) + if logentry.get('SYSLOG_PID'): + logelements[-1] += ("[%i]" % logentry['SYSLOG_PID']) + elif logentry.get('_PID'): + logelements[-1] += ("[%i]" % logentry['_PID']) logelements[-1] += ":" elif logentry.get('_COMM'): logelements.append(logentry['_COMM']) From 9269664350656bbdbab0131754322a2a7b4a9739 Mon Sep 17 00:00:00 2001 From: Guillaume FRANCOIS Date: Wed, 12 Nov 2014 10:30:28 +0100 Subject: [PATCH 12/31] Add ignoreregex to avoid warning on start --- config/filter.d/named-refused.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index 15eeedc4..eec3d667 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -38,6 +38,8 @@ failregex = ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: (vie ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$ ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ +ignoreregex = + # DEV Notes: # Trying to generalize the # structure which is general to capture general patterns in log From a6a2dc868b13be57fc986ca7c6986dac6e06185d Mon Sep 17 00:00:00 2001 From: Guillaume FRANCOIS Date: Wed, 12 Nov 2014 11:05:56 +0100 Subject: [PATCH 13/31] Add ignoreregex to avoid warning on start --- config/filter.d/recidive.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/filter.d/recidive.conf b/config/filter.d/recidive.conf index b38735ad..e2501cf6 100644 --- a/config/filter.d/recidive.conf +++ b/config/filter.d/recidive.conf @@ -29,6 +29,8 @@ _jailname = recidive failregex = ^(%(__prefix_line)s| %(_daemon)s%(__pid_re)s?:\s+)NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+\s*$ +ignoreregex = + [Init] journalmatch = _SYSTEMD_UNIT=fail2ban.service PRIORITY=5 From cad09d2df3f0a6f2fec52bdcf8b9dbb5f01eb593 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Nov 2014 03:17:47 +0100 Subject: [PATCH 14/31] BF: failregex declared direct in jail was joined to single line, (specifying of multiple expressions was not possible); feature request (gh-867): new options for jail introduced addfailregex/addignoreregex: extends regex specified in filter (opposite to failregex/ignoreregex that overwrites it); --- ChangeLog | 6 ++++++ fail2ban/client/jailreader.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1cd21884..7c0c61c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,8 +14,14 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee + * failregex declared in jail was joined to single line (specifying of multiple + expressions was not possible). + - New Features: + - new options for jail introduced addfailregex/addignoreregex: extends regex + specified in filter (opposite to failregex/ignoreregex that overwrites it) + see gh-867. - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 84cc5e2a..01186a1e 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -97,6 +97,8 @@ class JailReader(ConfigReader): ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], + ["string", "addfailregex", None], + ["string", "addignoreregex", None], ["string", "ignorecommand", None], ["string", "ignoreip", None], ["string", "filter", ""], @@ -201,11 +203,14 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "bantime", self.__opts[opt]]) elif opt == "usedns": stream.append(["set", self.__name, "usedns", self.__opts[opt]]) - elif opt == "failregex": - stream.append(["set", self.__name, "addfailregex", self.__opts[opt]]) + elif opt in ("failregex", "addfailregex"): + for regex in self.__opts[opt].split('\n'): + # Do not send a command if the rule is empty. + if regex != '': + stream.append(["set", self.__name, "addfailregex", regex]) elif opt == "ignorecommand": stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]]) - elif opt == "ignoreregex": + elif opt in ("ignoreregex", "addignoreregex"): for regex in self.__opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': From 143915212196a70d44b7ee30521977ff05c3be69 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Nov 2014 04:02:44 +0100 Subject: [PATCH 15/31] test cases extended (now correct) --- ChangeLog | 4 ++-- fail2ban/client/filterreader.py | 2 ++ fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/config/jail.conf | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7c0c61c5..e466cec0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,8 +14,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee - * failregex declared in jail was joined to single line (specifying of multiple - expressions was not possible). + * failregex declared direct in jail was joined to single line (specifying of + multiple expressions was not possible). - New Features: diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index fe657025..aff04f3b 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -50,6 +50,8 @@ class FilterReader(DefinitionInitConfigReader): def convert(self): stream = list() combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) + if not len(combinedopts): + return stream; opts = CommandAction.substituteRecursiveTags(combinedopts) if not opts: raise ValueError('recursive tag definitions unable to be resolved') diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 17a90cc8..31517f5d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -425,6 +425,17 @@ class JailsReaderTest(LogCaptureTestCase): self.maxDiff = None self.assertEqual(sorted(comm_commands), sorted([['add', 'emptyaction', 'auto'], + ['add', 'test1addfailregex', 'auto'], + ['set', 'test1addfailregex', 'addfailregex', 'failure 1 '], + ['set', 'test1addfailregex', 'addfailregex', 'failure 2 '], + ['set', 'test1addfailregex', 'addignoreregex', 'ignore 1 '], + ['set', 'test1addfailregex', 'addignoreregex', 'ignore 2 '], + ['set', 'test1addfailregex', 'addfailregex', ''], + ['start', 'test1addfailregex'], + ['add', 'test2failregex', 'auto'], + ['set', 'test2failregex', 'addfailregex', 'failure 1 '], + ['set', 'test2failregex', 'addfailregex', 'failure 2 '], + ['start', 'test2failregex'], ['add', 'missinglogfiles', 'auto'], ['set', 'missinglogfiles', 'addfailregex', ''], ['add', 'brokenaction', 'auto'], diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 0f6a28f0..c9ae6c54 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -13,6 +13,20 @@ failregex = ignoreregex = ignoreip = +[test1addfailregex] +enabled = true +filter = simple +addfailregex = failure 1 + failure 2 +addignoreregex = ignore 1 + ignore 2 + +[test2failregex] +enabled = true +filter = simple +failregex = failure 1 + failure 2 + [missinglogfiles] enabled = true logpath = /weapons/of/mass/destruction From d63b125877d56bff56a43c286d576a015d02c1f0 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Nov 2014 19:03:58 +0100 Subject: [PATCH 16/31] interpolation of config readers extended with `%(known/parameter)s`. (means last known option with name `parameter`). --- ChangeLog | 2 ++ fail2ban/client/configparserinc.py | 5 +++++ fail2ban/tests/clientreadertestcase.py | 5 +++++ fail2ban/tests/config/filter.d/test.conf | 6 ++++++ fail2ban/tests/config/filter.d/test.local | 7 +++++++ fail2ban/tests/config/jail.conf | 5 +++++ 6 files changed, 30 insertions(+) create mode 100644 fail2ban/tests/config/filter.d/test.conf create mode 100644 fail2ban/tests/config/filter.d/test.local diff --git a/ChangeLog b/ChangeLog index e466cec0..2f3bb9a9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,6 +19,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: + - new interpolation feature for config readers - `%(known/parameter)s`. + (means last known option with name `parameter`). - new options for jail introduced addfailregex/addignoreregex: extends regex specified in filter (opposite to failregex/ignoreregex that overwrites it) see gh-867. diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index d819281b..6299b906 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -226,6 +226,11 @@ after = 1.conf if isinstance(s, dict): s2 = alls.get(n) if isinstance(s2, dict): + # save previous known values, for possible using in local interpolations later: + for k, v in s2.iteritems(): + if not k.startswith('known/'): + s2['known/'+k] = v + # merge section s2.update(s) else: alls[n] = s.copy() diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 31517f5d..c684434d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -436,6 +436,11 @@ class JailsReaderTest(LogCaptureTestCase): ['set', 'test2failregex', 'addfailregex', 'failure 1 '], ['set', 'test2failregex', 'addfailregex', 'failure 2 '], ['start', 'test2failregex'], + ['add', 'test3known-interp', 'auto'], + ['set', 'test3known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) '], + ['set', 'test3known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) '], + ['set', 'test3known-interp', 'addfailregex', 'failure test 3 (jail.local) '], + ['start', 'test3known-interp'], ['add', 'missinglogfiles', 'auto'], ['set', 'missinglogfiles', 'addfailregex', ''], ['add', 'brokenaction', 'auto'], diff --git a/fail2ban/tests/config/filter.d/test.conf b/fail2ban/tests/config/filter.d/test.conf new file mode 100644 index 00000000..f09d3467 --- /dev/null +++ b/fail2ban/tests/config/filter.d/test.conf @@ -0,0 +1,6 @@ +#[INCLUDES] +#before = common.conf + +[Definition] +failregex = failure test 1 (filter.d/test.conf) + diff --git a/fail2ban/tests/config/filter.d/test.local b/fail2ban/tests/config/filter.d/test.local new file mode 100644 index 00000000..1b6cf55e --- /dev/null +++ b/fail2ban/tests/config/filter.d/test.local @@ -0,0 +1,7 @@ +#[INCLUDES] +#before = common.conf + +[Definition] +failregex = %(known/failregex)s + failure test 2 (filter.d/test.local) + diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index c9ae6c54..c79e255c 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -27,6 +27,11 @@ filter = simple failregex = failure 1 failure 2 +[test3known-interp] +enabled = true +filter = test +addfailregex = failure test 3 (jail.local) + [missinglogfiles] enabled = true logpath = /weapons/of/mass/destruction From 00c2ac4b034b68185926b0f7c8faaf9b3d021f18 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Nov 2014 19:50:52 +0100 Subject: [PATCH 17/31] python 2.6 compatibility: preventing RuntimeError: dictionary changed size during iteration. --- fail2ban/client/configparserinc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 6299b906..01fadb87 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -227,9 +227,11 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: + sk = {} for k, v in s2.iteritems(): if not k.startswith('known/'): - s2['known/'+k] = v + sk['known/'+k] = v + s2.update(sk) # merge section s2.update(s) else: From d8867807f560838e70375cc9ca90585179700fe6 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 28 Nov 2014 22:04:09 -0700 Subject: [PATCH 18/31] Separate php-url-fopen logpath by newline --- config/jail.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index d119d229..6a95aa12 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -302,7 +302,8 @@ logpath = %(nginx_error_log)s [php-url-fopen] port = http,https -logpath = %(nginx_access_log)s %(apache_access_log)s +logpath = %(nginx_access_log)s + %(apache_access_log)s [suhosin] From 9bab6d00094f01335269a89702fb5724324dc801 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 29 Nov 2014 09:52:25 -0500 Subject: [PATCH 19/31] Changelog entry for preceding fix --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1cd21884..620ab0f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * grep'ing for IP in *mail-whois-lines.conf should now match also at the begginning and EOL. Thanks Dean Lee + * jail.conf + - php-url-fopen: separate logpath entries by newline - New Features: From effdb450fc42ddf1169312b104e6f3678802442a Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 29 Nov 2014 04:09:53 +0100 Subject: [PATCH 20/31] better and scalable solution for gh-867 (and gh-868), using only name convention like %(known/failregex)s to add custom expressions, so no interface changes in jail.conf are necessary (for example see test-known-interp in test cases); --- ChangeLog | 8 ++--- fail2ban/client/configparserinc.py | 9 +++++ fail2ban/client/configreader.py | 4 +++ fail2ban/client/filterreader.py | 14 +++++--- fail2ban/client/jailreader.py | 28 ++++++++++----- fail2ban/tests/clientreadertestcase.py | 49 ++++++++++++-------------- fail2ban/tests/config/jail.conf | 19 ++-------- 7 files changed, 73 insertions(+), 58 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2f3bb9a9..fddf8751 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,10 +19,10 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: - - new interpolation feature for config readers - `%(known/parameter)s`. - (means last known option with name `parameter`). - - new options for jail introduced addfailregex/addignoreregex: extends regex - specified in filter (opposite to failregex/ignoreregex that overwrites it) + - New interpolation feature for config readers - `%(known/parameter)s`. + (means last known option with name `parameter`). This interpolation makes + possible to extend a stock filter or jail regexp in .local file + (opposite to simply set failregex/ignoreregex that overwrites it), see gh-867. - Enhancements: diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 01fadb87..387f0f2c 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -249,3 +249,12 @@ after = 1.conf return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') else: return SafeConfigParser.read(self, fileNamesFull) + + def merge_section(self, section, options, pref='known/'): + alls = self.get_sections() + sk = {} + for k, v in options.iteritems(): + if pref == '' or not k.startswith(pref): + sk[pref+k] = v + alls[section].update(sk) + diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b6c39628..ed1fbcde 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -116,6 +116,10 @@ class ConfigReader(): return self._cfg.has_section(sec) return False + def merge_section(self, *args, **kwargs): + if self._cfg is not None: + return self._cfg.merge_section(*args, **kwargs) + def options(self, *args): if self._cfg is not None: return self._cfg.options(*args) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index aff04f3b..669a5896 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -46,15 +46,21 @@ class FilterReader(DefinitionInitConfigReader): def getFile(self): return self.__file - - def convert(self): - stream = list() + + def getCombined(self): combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) if not len(combinedopts): - return stream; + return {}; opts = CommandAction.substituteRecursiveTags(combinedopts) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + return opts; + + def convert(self): + stream = list() + opts = self.getCombined() + if not len(opts): + return stream; for opt, value in opts.iteritems(): if opt == "failregex": for regex in value.split('\n'): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 01186a1e..340508c0 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -87,6 +87,8 @@ class JailReader(ConfigReader): return pathList def getOptions(self): + opts1st = [["bool", "enabled", False], + ["string", "filter", ""]] opts = [["bool", "enabled", False], ["string", "logpath", None], ["string", "logencoding", None], @@ -97,13 +99,13 @@ class JailReader(ConfigReader): ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], - ["string", "addfailregex", None], - ["string", "addignoreregex", None], ["string", "ignorecommand", None], ["string", "ignoreip", None], ["string", "filter", ""], ["string", "action", ""]] - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + + # Read first options only needed for merge defaults ('known/...' from filter): + self.__opts = ConfigReader.getOptions(self, self.__name, opts1st) if not self.__opts: return False @@ -115,14 +117,24 @@ class JailReader(ConfigReader): self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() - if ret: - self.__filter.getOptions(self.__opts) - else: + # merge options from filter as 'known/...': + self.__filter.getOptions(self.__opts) + ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') + if not ret: logSys.error("Unable to read the filter") return False else: self.__filter = None logSys.warning("No filter set for jail %s" % self.__name) + + # Read second all options (so variables like %(known/param) can be interpolated): + self.__opts = ConfigReader.getOptions(self, self.__name, opts) + if not self.__opts: + return False + + # cumulate filter options again (ignore given in jail): + if self.__filter: + self.__filter.getOptions(self.__opts) # Read action for act in self.__opts["action"].split('\n'): @@ -203,14 +215,14 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "bantime", self.__opts[opt]]) elif opt == "usedns": stream.append(["set", self.__name, "usedns", self.__opts[opt]]) - elif opt in ("failregex", "addfailregex"): + elif opt == "failregex": for regex in self.__opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self.__name, "addfailregex", regex]) elif opt == "ignorecommand": stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]]) - elif opt in ("ignoreregex", "addignoreregex"): + elif opt == "ignoreregex": for regex in self.__opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index c684434d..4f77a698 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -155,12 +155,16 @@ c = d ;in line comment class JailReaderTest(LogCaptureTestCase): + def __init__(self, *args, **kwargs): + super(JailReaderTest, self).__init__(*args, **kwargs) + self.__share_cfg = {} + def testIncorrectJail(self): - jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR) + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg) self.assertRaises(ValueError, jail.read) def testJailActionEmpty(self): - jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG) + jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -168,7 +172,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged('No actions were defined for emptyaction')) def testJailActionFilterMissing(self): - jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG) + jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -176,7 +180,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(self._is_logged('Unable to read the filter')) def TODOtestJailActionBrokenDef(self): - jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG) + jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -187,7 +191,7 @@ class JailReaderTest(LogCaptureTestCase): if STOCK: def testStockSSHJail(self): - jail = JailReader('sshd', basedir=CONFIG_DIR) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -411,13 +415,17 @@ class JailsReaderTestCache(LogCaptureTestCase): class JailsReaderTest(LogCaptureTestCase): + def __init__(self, *args, **kwargs): + super(JailsReaderTest, self).__init__(*args, **kwargs) + self.__share_cfg = {} + def testProvidingBadBasedir(self): if not os.path.exists('/XXX'): reader = JailsReader(basedir='/XXX') self.assertRaises(ValueError, reader.read) def testReadTestJailConf(self): - jails = JailsReader(basedir=IMPERFECT_CONFIG) + jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jails.read()) self.assertFalse(jails.getOptions()) self.assertRaises(ValueError, jails.convert) @@ -425,22 +433,11 @@ class JailsReaderTest(LogCaptureTestCase): self.maxDiff = None self.assertEqual(sorted(comm_commands), sorted([['add', 'emptyaction', 'auto'], - ['add', 'test1addfailregex', 'auto'], - ['set', 'test1addfailregex', 'addfailregex', 'failure 1 '], - ['set', 'test1addfailregex', 'addfailregex', 'failure 2 '], - ['set', 'test1addfailregex', 'addignoreregex', 'ignore 1 '], - ['set', 'test1addfailregex', 'addignoreregex', 'ignore 2 '], - ['set', 'test1addfailregex', 'addfailregex', ''], - ['start', 'test1addfailregex'], - ['add', 'test2failregex', 'auto'], - ['set', 'test2failregex', 'addfailregex', 'failure 1 '], - ['set', 'test2failregex', 'addfailregex', 'failure 2 '], - ['start', 'test2failregex'], - ['add', 'test3known-interp', 'auto'], - ['set', 'test3known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) '], - ['set', 'test3known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) '], - ['set', 'test3known-interp', 'addfailregex', 'failure test 3 (jail.local) '], - ['start', 'test3known-interp'], + ['add', 'test-known-interp', 'auto'], + ['set', 'test-known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) '], + ['set', 'test-known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) '], + ['set', 'test-known-interp', 'addfailregex', 'failure test 3 (jail.local) '], + ['start', 'test-known-interp'], ['add', 'missinglogfiles', 'auto'], ['set', 'missinglogfiles', 'addfailregex', ''], ['add', 'brokenaction', 'auto'], @@ -463,7 +460,7 @@ class JailsReaderTest(LogCaptureTestCase): if STOCK: def testReadStockJailConf(self): - jails = JailsReader(basedir=CONFIG_DIR) # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert() @@ -524,7 +521,7 @@ class JailsReaderTest(LogCaptureTestCase): # Verify that all filters found under config/ have a jail def testReadStockJailFilterComplete(self): - jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) + jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine # grab all filter names @@ -541,7 +538,7 @@ class JailsReaderTest(LogCaptureTestCase): def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises # on users' systems when enabling shipped jails - jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert(allow_no_files=True) @@ -636,7 +633,7 @@ action = testaction1[actname=test1] filter = testfilter1 """) jailfd.close() - jails = JailsReader(basedir=basedir) + jails = JailsReader(basedir=basedir, share_config=self.__share_cfg) self.assertTrue(jails.read()) self.assertTrue(jails.getOptions()) comm_commands = jails.convert(allow_no_files=True) diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index c79e255c..bf1dea45 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -13,24 +13,11 @@ failregex = ignoreregex = ignoreip = -[test1addfailregex] -enabled = true -filter = simple -addfailregex = failure 1 - failure 2 -addignoreregex = ignore 1 - ignore 2 - -[test2failregex] -enabled = true -filter = simple -failregex = failure 1 - failure 2 - -[test3known-interp] +[test-known-interp] enabled = true filter = test -addfailregex = failure test 3 (jail.local) +failregex = %(known/failregex)s + failure test 3 (jail.local) [missinglogfiles] enabled = true From d62b0467048d7d6ec57e66b9fc9a65acfded0010 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Dec 2014 14:45:46 +0100 Subject: [PATCH 21/31] testSetupInstallRoot will be always skipped, because of "wrong" location of 'setup.py'; --- fail2ban/tests/misctestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 64b6b65e..4bb4b02f 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -67,7 +67,7 @@ def recursive_glob(treeroot, pattern): class SetupTest(unittest.TestCase): def setUp(self): - setup = os.path.join(os.path.dirname(__file__), '..', 'setup.py') + setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None if not self.setup and sys.version_info >= (2,7): # running not out of the source raise unittest.SkipTest( From 7d4f071d4b9b0b67310fb72b3107b1e977ae5c1a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Dec 2014 15:25:27 +0100 Subject: [PATCH 22/31] small fix: no cover for failed case --- fail2ban/tests/misctestcase.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 4bb4b02f..afc11ed9 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -55,21 +55,12 @@ class HelpersTest(unittest.TestCase): # might be fragile due to ' vs " self.assertEqual(args, "('Very bad', None)") -# based on -# http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python -def recursive_glob(treeroot, pattern): - results = [] - for base, dirs, files in os.walk(treeroot): - goodfiles = fnmatch.filter(dirs + files, pattern) - results.extend(os.path.join(base, f) for f in goodfiles) - return results - class SetupTest(unittest.TestCase): def setUp(self): setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None - if not self.setup and sys.version_info >= (2,7): # running not out of the source + if not self.setup and sys.version_info >= (2,7): # pragma: no cover - running not out of the source raise unittest.SkipTest( "Seems to be running not out of source distribution" " -- cannot locate setup.py") @@ -80,9 +71,6 @@ class SetupTest(unittest.TestCase): os.system("%s %s install --root=%s >/dev/null" % (sys.executable, self.setup, tmp)) - def addpath(l): - return [os.path.join(tmp, x) for x in l] - def strippath(l): return [x[len(tmp)+1:] for x in l] @@ -90,10 +78,20 @@ class SetupTest(unittest.TestCase): need = ['etc', 'usr', 'var'] # if anything is missing - if set(need).difference(got): + if set(need).difference(got): # pragma: no cover # below code was actually to print out not missing but # rather files in 'excess'. Left in place in case we # decide to revert to such more strict test + + # based on + # http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python + def recursive_glob(treeroot, pattern): + results = [] + for base, dirs, files in os.walk(treeroot): + goodfiles = fnmatch.filter(dirs + files, pattern) + results.extend(os.path.join(base, f) for f in goodfiles) + return results + files = {} for missing in set(got).difference(need): missing_full = os.path.join(tmp, missing) From ccc986b7d8bb0d615c85fefd07aaedc04af33743 Mon Sep 17 00:00:00 2001 From: bes-internal Date: Mon, 1 Dec 2014 13:12:53 +0300 Subject: [PATCH 23/31] exim filter: correct failregex for exim with extended log options incoming_interface, incoming_port, outgoing_port --- ChangeLog | 2 ++ config/filter.d/exim.conf | 4 ++-- fail2ban/tests/files/logs/exim | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 08cf38cc..690b168a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - php-url-fopen: separate logpath entries by newline * failregex declared direct in jail was joined to single line (specifying of multiple expressions was not possible). + * filters.d/exim.conf - cover different settings of exim logs + details. Thanks bes.internal - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index b5028f0e..11fd03d1 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -14,10 +14,10 @@ before = exim-common.conf [Definition] failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ - ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[\](:\d+)?( I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$ - ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ + ^%(pid)s SMTP call from \S+ \[\](:\d+)? (I=\[\S+\](:\d+)? )?dropped: too many nonmail commands \(last was "\S+"\)\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 84aedcfa..c015eb29 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -40,3 +40,6 @@ # failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" } 2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner) + +# failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" } +2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster) From a71a64733a747ddbac5fb9515afc6afb6ffd95df Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 5 Dec 2014 16:39:17 +0100 Subject: [PATCH 24/31] clean all after test setup (removes a build directory in current root of fail2ban) --- fail2ban/tests/misctestcase.py | 76 ++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index afc11ed9..2f69aef0 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -68,49 +68,53 @@ class SetupTest(unittest.TestCase): def testSetupInstallRoot(self): if not self.setup: return # if verbose skip didn't work out tmp = tempfile.mkdtemp() - os.system("%s %s install --root=%s >/dev/null" - % (sys.executable, self.setup, tmp)) + try: + os.system("%s %s install --root=%s >/dev/null" + % (sys.executable, self.setup, tmp)) - def strippath(l): - return [x[len(tmp)+1:] for x in l] + def strippath(l): + return [x[len(tmp)+1:] for x in l] - got = strippath(sorted(glob('%s/*' % tmp))) - need = ['etc', 'usr', 'var'] + got = strippath(sorted(glob('%s/*' % tmp))) + need = ['etc', 'usr', 'var'] - # if anything is missing - if set(need).difference(got): # pragma: no cover - # below code was actually to print out not missing but - # rather files in 'excess'. Left in place in case we - # decide to revert to such more strict test + # if anything is missing + if set(need).difference(got): # pragma: no cover + # below code was actually to print out not missing but + # rather files in 'excess'. Left in place in case we + # decide to revert to such more strict test - # based on - # http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python - def recursive_glob(treeroot, pattern): - results = [] - for base, dirs, files in os.walk(treeroot): - goodfiles = fnmatch.filter(dirs + files, pattern) - results.extend(os.path.join(base, f) for f in goodfiles) - return results + # based on + # http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python + def recursive_glob(treeroot, pattern): + results = [] + for base, dirs, files in os.walk(treeroot): + goodfiles = fnmatch.filter(dirs + files, pattern) + results.extend(os.path.join(base, f) for f in goodfiles) + return results - files = {} - for missing in set(got).difference(need): - missing_full = os.path.join(tmp, missing) - files[missing] = os.path.exists(missing_full) \ - and strippath(recursive_glob(missing_full, '*')) or None + files = {} + for missing in set(got).difference(need): + missing_full = os.path.join(tmp, missing) + files[missing] = os.path.exists(missing_full) \ + and strippath(recursive_glob(missing_full, '*')) or None - self.assertEqual( - got, need, - msg="Got: %s Needed: %s under %s. Files under new paths: %s" - % (got, need, tmp, files)) + self.assertEqual( + got, need, + msg="Got: %s Needed: %s under %s. Files under new paths: %s" + % (got, need, tmp, files)) - # Assure presence of some files we expect to see in the installation - for f in ('etc/fail2ban/fail2ban.conf', - 'etc/fail2ban/jail.conf'): - self.assertTrue(os.path.exists(os.path.join(tmp, f)), - msg="Can't find %s" % f) - - # clean up - shutil.rmtree(tmp) + # Assure presence of some files we expect to see in the installation + for f in ('etc/fail2ban/fail2ban.conf', + 'etc/fail2ban/jail.conf'): + self.assertTrue(os.path.exists(os.path.join(tmp, f)), + msg="Can't find %s" % f) + finally: + # clean up + shutil.rmtree(tmp) + # remove build directory + os.system("%s %s clean --all >/dev/null" + % (sys.executable, self.setup)) class TestsUtilsTest(unittest.TestCase): From 81b3dbde1deab56955748277ef682523f6d862b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Thu, 11 Dec 2014 00:10:37 +0100 Subject: [PATCH 25/31] postfix-sasl failregex case insensitive --- config/filter.d/postfix-sasl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index 35b064d3..e038b695 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -9,7 +9,7 @@ before = common.conf _daemon = postfix/(submission/)?smtp(d|s) -failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$ +failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$ ignoreregex = From 190f55b06e531f06b425505909de9c8561ae6dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Thu, 11 Dec 2014 01:34:20 +0100 Subject: [PATCH 26/31] Added an item to "Fixes" --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 690b168a..c3c314af 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released multiple expressions was not possible). * filters.d/exim.conf - cover different settings of exim logs details. Thanks bes.internal + * filter.d/postfix-sasl.conf - failregex is now case insensitive - New Features: - New interpolation feature for config readers - `%(known/parameter)s`. From a9b6a3754b723001b26a87a2b3b1079754b83bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Thu, 11 Dec 2014 21:01:52 +0100 Subject: [PATCH 27/31] downcase example --- fail2ban/tests/files/logs/postfix-sasl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl index 697ac424..46c1e9da 100644 --- a/fail2ban/tests/files/logs/postfix-sasl +++ b/fail2ban/tests/files/logs/postfix-sasl @@ -8,3 +8,7 @@ Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LO #3 Example from postfix post-debian changes to rename to add "submission" to syslog name # failJSON: { "time": "2004-09-06T00:44:56", "match": true , "host": "82.221.106.233" } Sep 6 00:44:56 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL LOGIN authentication failed: UGFzc3dvcmQ6 + +#4 Example from postfix post-debian changes to rename to add "submission" to syslog name + downcase +# failJSON: { "time": "2004-09-06T00:44:57", "match": true , "host": "82.221.106.233" } +Sep 6 00:44:57 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL login authentication failed: UGFzc3dvcmQ6 From cc89649d04f3ec9e6613f31b9fa8769881ef5428 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 21 Dec 2014 21:40:58 -0500 Subject: [PATCH 28/31] BF: adjusted for new IP of example.com Conflicts: fail2ban/tests/filtertestcase.py --- fail2ban/tests/files/testcase-usedns.log | 2 +- fail2ban/tests/filtertestcase.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fail2ban/tests/files/testcase-usedns.log b/fail2ban/tests/files/testcase-usedns.log index a91fd7ac..de484127 100644 --- a/fail2ban/tests/files/testcase-usedns.log +++ b/fail2ban/tests/files/testcase-usedns.log @@ -1,2 +1,2 @@ Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2 -Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2 +Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 1fa3116e..47e88067 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -888,12 +888,12 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) - output_yes = ('93.184.216.119', 2, 1124013539.0, + output_yes = ('93.184.216.34', 2, 1124013539.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2']) + u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']) - output_no = ('93.184.216.119', 1, 1124013539.0, - [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2']) + output_no = ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']) # Actually no exception would be raised -- it will be just set to 'no' #self.assertRaises(ValueError, @@ -993,9 +993,9 @@ class DNSUtilsTests(unittest.TestCase): res = DNSUtils.textToIp('www.example.com', 'no') self.assertEqual(res, []) res = DNSUtils.textToIp('www.example.com', 'warn') - self.assertEqual(res, ['93.184.216.119']) + self.assertEqual(res, ['93.184.216.34']) res = DNSUtils.textToIp('www.example.com', 'yes') - self.assertEqual(res, ['93.184.216.119']) + self.assertEqual(res, ['93.184.216.34']) def testTextToIp(self): # Test hostnames @@ -1007,7 +1007,7 @@ class DNSUtilsTests(unittest.TestCase): for s in hostnames: res = DNSUtils.textToIp(s, 'yes') if s == 'www.example.com': - self.assertEqual(res, ['93.184.216.119']) + self.assertEqual(res, ['93.184.216.34']) else: self.assertEqual(res, []) From 971c3d93feefb6f189faaa13faf93f890d40b9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Wed, 24 Dec 2014 14:45:48 +0100 Subject: [PATCH 29/31] Monit config Copy it to `/etc/monit/monitrc.d/fail2ban` More info: https://github.com/szepeviktor/debian-server-tools/blob/master/monitoring/monit/monit-debian-setup.sh#L113 --- files/monit/fail2ban | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 files/monit/fail2ban diff --git a/files/monit/fail2ban b/files/monit/fail2ban new file mode 100644 index 00000000..8e6c9419 --- /dev/null +++ b/files/monit/fail2ban @@ -0,0 +1,9 @@ +check process fail2ban with pidfile /var/run/fail2ban/fail2ban.pid + group services + start program = "/etc/init.d/fail2ban force-start" + stop program = "/etc/init.d/fail2ban stop || :" + if failed unixsocket /var/run/fail2ban/fail2ban.sock then restart + if 5 restarts within 5 cycles then timeout + +check file fail2ban_log with path /var/log/fail2ban.log + if match "ERROR|WARNING" then alert From 10f68f494605fac41310ffc9a4bfc48158535013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Wed, 24 Dec 2014 15:00:25 +0100 Subject: [PATCH 30/31] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 690b168a..db419c3b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), see gh-867. + - Monit config for fail2ban in /files/monit - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 From d65c4f8f5d9ef0a7748ab6ad3149514225fe78fa Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 30 Dec 2014 16:44:51 -0500 Subject: [PATCH 31/31] moved debian's initd file to files/debian-initd from debian branch --- ChangeLog | 2 + MANIFEST | 1 + files/debian-initd | 248 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100755 files/debian-initd diff --git a/ChangeLog b/ChangeLog index c80dac5a..853f603e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 + * files/debian-initd migrated from the debian branch and should be + suitable for manual installations now (thanks Juan Karlo de Guzman) ver. 0.9.1 (2014/10/29) - better, faster, stronger diff --git a/MANIFEST b/MANIFEST index eeb13eef..c6de80fd 100644 --- a/MANIFEST +++ b/MANIFEST @@ -328,6 +328,7 @@ man/fail2ban-server.h2m man/fail2ban-regex.1 man/fail2ban-regex.h2m man/generate-man +files/debian-initd files/gentoo-initd files/gentoo-confd files/redhat-initd diff --git a/files/debian-initd b/files/debian-initd new file mode 100755 index 00000000..d9bb3f50 --- /dev/null +++ b/files/debian-initd @@ -0,0 +1,248 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: fail2ban +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Should-Start: $time $network $syslog iptables firehol shorewall ipmasq arno-iptables-firewall iptables-persistent ferm +# Should-Stop: $network $syslog iptables firehol shorewall ipmasq arno-iptables-firewall iptables-persistent ferm +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start/stop fail2ban +# Description: Start/stop fail2ban, a daemon scanning the log files and +# banning potential attackers. +### END INIT INFO + +# Author: Aaron Isotton +# Modified: by Yaroslav Halchenko +# reindented + minor corrections + to work on sarge without modifications +# Modified: by Glenn Aaldering +# added exit codes for status command +# Modified: by Juan Karlo de Guzman +# corrected the DAEMON's path and the SOCKFILE +# rename this file: (sudo) mv /etc/init.d/fail2ban.init /etc/init.d/fail2ban +# same with the logrotate file: (sudo) mv /etc/logrotate.d/fail2ban.logrotate /etc/logrotate.d/fail2ban +# +PATH=/usr/sbin:/usr/bin:/sbin:/bin +DESC="authentication failure monitor" +NAME=fail2ban + +# fail2ban-client is not a daemon itself but starts a daemon and +# loads its with configuration +DAEMON=/usr/local/bin/$NAME-client +SCRIPTNAME=/etc/init.d/$NAME + +# Ad-hoc way to parse out socket file name +SOCKFILE=`grep -h '^[^#]*socket *=' /etc/$NAME/$NAME.conf /etc/$NAME/$NAME.local 2>/dev/null \ + | tail -n 1 | sed -e 's/.*socket *= *//g' -e 's/ *$//g'` +[ -z "$SOCKFILE" ] && SOCKFILE='/var/run/fail2ban.sock' + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Run as root by default. +FAIL2BAN_USER=root + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME +DAEMON_ARGS="$FAIL2BAN_OPTS" + +# Load the VERBOSE setting and other rcS variables +[ -f /etc/default/rcS ] && . /etc/default/rcS + +# Predefine what can be missing from lsb source later on -- necessary to run +# on sarge. Just present it in a bit more compact way from what was shipped +log_daemon_msg () { + [ -z "$1" ] && return 1 + echo -n "$1:" + [ -z "$2" ] || echo -n " $2" +} + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. +# Actually has to (>=2.0-7) present in sarge. log_daemon_msg is predefined +# so we must be ok +. /lib/lsb/init-functions + +# +# Shortcut function for abnormal init script interruption +# +report_bug() +{ + echo $* + echo "Please submit a bug report to Debian BTS (reportbug fail2ban)" + exit 1 +} + +# +# Helper function to check if socket is present, which is often left after +# abnormal exit of fail2ban and needs to be removed +# +check_socket() +{ + # Return + # 0 if socket is present and readable + # 1 if socket file is not present + # 2 if socket file is present but not readable + # 3 if socket file is present but is not a socket + [ -e "$SOCKFILE" ] || return 1 + [ -r "$SOCKFILE" ] || return 2 + [ -S "$SOCKFILE" ] || return 3 + return 0 +} + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + do_status && return 1 + + if [ -e "$SOCKFILE" ]; then + log_failure_msg "Socket file $SOCKFILE is present" + [ "$1" = "force-start" ] \ + && log_success_msg "Starting anyway as requested" \ + || return 2 + DAEMON_ARGS="$DAEMON_ARGS -x" + fi + + # Assure that /var/run/fail2ban exists + [ -d /var/run/fail2ban ] || mkdir -p /var/run/fail2ban + + if [ "$FAIL2BAN_USER" != "root" ]; then + # Make the socket directory, IP lists and fail2ban log + # files writable by fail2ban + chown "$FAIL2BAN_USER" /var/run/fail2ban + # Create the logfile if it doesn't exist + touch /var/log/fail2ban.log + chown "$FAIL2BAN_USER" /var/log/fail2ban.log + find /proc/net/xt_recent -name 'fail2ban-*' -exec chown "$FAIL2BAN_USER" {} \; + fi + + start-stop-daemon --start --quiet --chuid "$FAIL2BAN_USER" --exec $DAEMON -- \ + $DAEMON_ARGS start > /dev/null\ + || return 2 + + return 0 +} + + +# +# Function that checks the status of fail2ban and returns +# corresponding code +# +do_status() +{ + $DAEMON ping > /dev/null 2>&1 + return $? +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + $DAEMON status > /dev/null 2>&1 || return 1 + $DAEMON stop > /dev/null || return 2 + + # now we need actually to wait a bit since it might take time + # for server to react on client's stop request. Especially + # important for restart command on slow boxes + count=1 + while do_status && [ $count -lt 60 ]; do + sleep 1 + count=$(($count+1)) + done + [ $count -lt 60 ] || return 3 # failed to stop + + return 0 +} + +# +# Function to reload configuration +# +do_reload() { + $DAEMON reload > /dev/null && return 0 || return 1 + return 0 +} + +# yoh: +# shortcut function to don't duplicate case statements and to don't use +# bashisms (arrays). Fixes #368218 +# +log_end_msg_wrapper() +{ + if [ "$3" != "no" ]; then + [ $1 -lt $2 ] && value=0 || value=1 + log_end_msg $value + fi +} + +command="$1" +case "$command" in + start|force-start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start "$command" + log_end_msg_wrapper $? 2 "$VERBOSE" + ;; + + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + log_end_msg_wrapper $? 2 "$VERBOSE" + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + log_end_msg_wrapper $? 1 "always" + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + + reload|force-reload) + log_daemon_msg "Reloading $DESC" "$NAME" + do_reload + log_end_msg $? + ;; + + status) + log_daemon_msg "Status of $DESC" + do_status + case $? in + 0) log_success_msg " $NAME is running" ;; + 255) + check_socket + case $? in + 1) log_failure_msg " $NAME is not running" && exit 3 ;; + 0) log_failure_msg " $NAME is not running but $SOCKFILE exists" && exit 3 ;; + 2) log_failure_msg " $SOCKFILE not readable, status of $NAME is unknown" && exit 3 ;; + 3) log_failure_msg " $SOCKFILE exists but not a socket, status of $NAME is unknown" && exit 3 ;; + *) report_bug "Unknown return code from $NAME:check_socket." && exit 4 ;; + esac + ;; + *) report_bug "Unknown $NAME status code" && exit 4 + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|force-start|stop|restart|force-reload|status}" >&2 + exit 3 + ;; +esac + +: