diff --git a/ChangeLog b/ChangeLog index 8af2c6f8..204c9559 100644 --- a/ChangeLog +++ b/ChangeLog @@ -112,6 +112,11 @@ releases. * New forward compatibility method assertRaisesRegexp (normally python >= 2.7). Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged are test covered now +* Jail configuration extended with new syntax to pass options to the backend (see gh-1408), + examples: + - `backend = systemd[journalpath=/run/log/journal/machine-1]` + - `backend = systemd[journalfiles="/run/log/journal/machine-1/system.journal, /run/log/journal/machine-1/user.journal"]` + - `backend = systemd[journalflags=2]` ver. 0.9.5 (2016/07/15) - old-not-obsolete diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 5f2e64b2..13f611f7 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -197,7 +197,7 @@ class JailReader(ConfigReader): stream = [] for opt, value in self.__opts.iteritems(): if opt == "logpath" and \ - self.__opts.get('backend', None) != "systemd": + not self.__opts.get('backend', None).startswith("systemd"): found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index d0ebec95..9c4c7865 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -33,7 +33,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': from .failmanager import FailManagerEmpty from .filter import JournalFilter from .mytime import MyTime -from ..helpers import getLogger +from ..helpers import getLogger, logging, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) @@ -54,14 +54,45 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # @param jail the jail object def __init__(self, jail, **kwargs): + jrnlargs = FilterSystemd._getJournalArgs(kwargs) JournalFilter.__init__(self, jail, **kwargs) - self.__modified = False + self.__modified = 0 # Initialise systemd-journal connection - self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x}) + self.__journal = journal.Reader(**jrnlargs) self.__matches = [] self.setDatePattern(None) + self.ticks = 0 logSys.debug("Created FilterSystemd") + @staticmethod + def _getJournalArgs(kwargs): + args = {'converters':{'__CURSOR': lambda x: x}} + try: + args['path'] = kwargs.pop('journalpath') + except KeyError: + pass + + try: + args['files'] = kwargs.pop('journalfiles') + except KeyError: + pass + else: + import glob + p = args['files'] + if not isinstance(p, (list, set, tuple)): + p = splitwords(p) + files = [] + for p in p: + files.extend(glob.glob(p)) + args['files'] = list(set(files)) + + try: + args['flags'] = kwargs.pop('journalflags') + except KeyError: + pass + + return args + ## # Add a journal match filters from list structure # @@ -207,6 +238,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover return (('', date.isoformat(), logline), time.mktime(date.timetuple()) + date.microsecond/1.0E6) + def seekToTime(self, date): + if not isinstance(date, datetime.datetime): + date = datetime.datetime.fromtimestamp(date) + self.__journal.seek_realtime(date) + ## # Main loop. # @@ -224,7 +260,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # Seek to now - findtime in journal start_time = datetime.datetime.now() - \ datetime.timedelta(seconds=int(self.getFindTime())) - self.__journal.seek_realtime(start_time) + self.seekToTime(start_time) # Move back one entry to ensure do not end up in dead space # if start time beyond end of journal try: @@ -233,29 +269,38 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover pass # Reading failure, so safe to ignore while self.active: - if not self.idle: - while self.active: - try: - logentry = self.__journal.get_next() - except OSError: - logSys.warning( - "Error reading line from systemd journal") - continue - if logentry: - self.processLineAndAdd( - *self.formatJournalEntry(logentry)) - self.__modified = True - else: - break - if self.__modified: - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False + # wait for records (or for timeout in sleeptime seconds): self.__journal.wait(self.sleeptime) + if self.idle: + # because journal.wait will returns immediatelly if we have records in journal, + # just wait a little bit here for not idle, to prevent hi-load: + time.sleep(self.sleeptime) + continue + self.__modified = 0 + while self.active: + logentry = None + try: + logentry = self.__journal.get_next() + except OSError as e: + logSys.error("Error reading line from systemd journal: %s", + e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) + self.ticks += 1 + if logentry: + self.processLineAndAdd( + *self.formatJournalEntry(logentry)) + self.__modified += 1 + if self.__modified >= 100: # todo: should be configurable + break + else: + break + if self.__modified: + try: + while True: + ticket = self.failManager.toBan() + self.jail.putFailTicket(ticket) + except FailManagerEmpty: + self.failManager.cleanup(MyTime.time()) + logSys.debug((self.jail is not None and self.jail.name or "jailless") +" filter terminated") return True diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 6345d300..7659a7ad 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -27,6 +27,7 @@ import logging import Queue from .actions import Actions +from ..client.jailreader import JailReader from ..helpers import getLogger # Gets the instance of the logger. @@ -83,6 +84,7 @@ class Jail(object): return "%s(%r)" % (self.__class__.__name__, self.name) def _setBackend(self, backend): + backend, beArgs = JailReader.extractOptions(backend) backend = backend.lower() # to assure consistent matching backends = self._BACKENDS @@ -99,7 +101,7 @@ class Jail(object): for b in backends: initmethod = getattr(self, '_init%s' % b.capitalize()) try: - initmethod() + initmethod(**beArgs) if backend != 'auto' and b != backend: logSys.warning("Could only initiated %r backend whenever " "%r was requested" % (b, backend)) @@ -119,28 +121,28 @@ class Jail(object): raise RuntimeError( "Failed to initialize any backend for Jail %r" % self.name) - def _initPolling(self): + def _initPolling(self, **kwargs): from filterpoll import FilterPoll - logSys.info("Jail '%s' uses poller" % self.name) - self.__filter = FilterPoll(self) + logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs)) + self.__filter = FilterPoll(self, **kwargs) - def _initGamin(self): + def _initGamin(self, **kwargs): # Try to import gamin from filtergamin import FilterGamin - logSys.info("Jail '%s' uses Gamin" % self.name) - self.__filter = FilterGamin(self) + logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs)) + self.__filter = FilterGamin(self, **kwargs) - def _initPyinotify(self): + def _initPyinotify(self, **kwargs): # Try to import pyinotify from filterpyinotify import FilterPyinotify - logSys.info("Jail '%s' uses pyinotify" % self.name) - self.__filter = FilterPyinotify(self) + logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs)) + self.__filter = FilterPyinotify(self, **kwargs) - def _initSystemd(self): # pragma: systemd no cover + def _initSystemd(self, **kwargs): # pragma: systemd no cover # Try to import systemd from filtersystemd import FilterSystemd - logSys.info("Jail '%s' uses systemd" % self.name) - self.__filter = FilterSystemd(self) + logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs)) + self.__filter = FilterSystemd(self, **kwargs) @property def name(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 39fd0daa..239989b7 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -37,7 +37,7 @@ from ..client.actionreader import ActionReader from ..client.configurator import Configurator from ..server.mytime import MyTime from ..version import version -from .utils import LogCaptureTestCase +from .utils import LogCaptureTestCase, with_tmpdir TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") TEST_FILES_DIR_SHARE_CFG = {} @@ -97,9 +97,8 @@ option = %s if not os.access(f, os.R_OK): self.assertFalse(self.c.read('d')) # should not be readable BUT present else: - # SkipTest introduced only in 2.7 thus can't yet use generally - # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) - pass + import platform + raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform.platform()) def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet @@ -298,8 +297,8 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) - def testGlob(self): - d = tempfile.mkdtemp(prefix="f2b-temp") + @with_tmpdir + def testGlob(self, d): # Generate few files # regular file f1 = os.path.join(d, 'f1') @@ -314,9 +313,6 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(JailReader._glob(f2), []) self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) - os.remove(f1) - os.remove(f2) - os.rmdir(d) class FilterReaderTest(unittest.TestCase): @@ -448,11 +444,11 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt += 1 return cnt - def testTestJailConfCache(self): + @with_tmpdir + def testTestJailConfCache(self, basedir): unittest.F2B.SkipIfFast() saved_ll = configparserinc.logLevel configparserinc.logLevel = logging.DEBUG - basedir = tempfile.mkdtemp("fail2ban_conf") try: shutil.rmtree(basedir) shutil.copytree(CONFIG_DIR, basedir) @@ -484,7 +480,6 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: - shutil.rmtree(basedir) configparserinc.logLevel = saved_ll @@ -739,8 +734,8 @@ class JailsReaderTest(LogCaptureTestCase): self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp') self.assertEqual(configurator.getBaseDir(), CONFIG_DIR) - def testMultipleSameAction(self): - basedir = tempfile.mkdtemp("fail2ban_conf") + @with_tmpdir + def testMultipleSameAction(self, basedir): os.mkdir(os.path.join(basedir, "filter.d")) os.mkdir(os.path.join(basedir, "action.d")) open(os.path.join(basedir, "action.d", "testaction1.conf"), 'w').close() @@ -769,4 +764,33 @@ filter = testfilter1 # Python actions should not be passed `actname` self.assertEqual(add_actions[-1][-1], "{}") - shutil.rmtree(basedir) + def testLogPathFileFilterBackend(self): + self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", + self._testLogPath, backend='polling') + + def testLogPathSystemdBackend(self): + try: # pragma: systemd no cover + from ..server.filtersystemd import FilterSystemd + except Exception, e: # pragma: no cover + raise unittest.SkipTest("systemd python interface not available") + self._testLogPath(backend='systemd') + self._testLogPath(backend='systemd[journalflags=2]') + + @with_tmpdir + def _testLogPath(self, basedir, backend): + jailfd = open(os.path.join(basedir, "jail.conf"), 'w') + jailfd.write(""" +[testjail1] +enabled = true +backend = %s +logpath = %s/not/exist.log + /this/path/should/not/exist.log +action = +filter = +failregex = test +""" % (backend, basedir)) + jailfd.close() + jails = JailsReader(basedir=basedir) + self.assertTrue(jails.read()) + self.assertTrue(jails.getOptions()) + jails.convert() diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index d039c920..02d2c79b 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -55,12 +55,10 @@ class DatabaseTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" super(DatabaseTest, self).setUp() - if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover + if Fail2BanDb is None: # pragma: no cover raise unittest.SkipTest( "Unable to import fail2ban database module as sqlite is not " "available.") - elif Fail2BanDb is None: - return self.dbFilename = None if not unittest.F2B.memory_db: _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 15058d76..0f338fbe 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -932,11 +932,16 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover super(MonitorJournalFailures, self).setUp() self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log") self.jail = DummyJail() - self.filter = Filter_(self.jail) + self.filter = None # UUID used to ensure that only meeages generated # as part of this test are picked up by the filter self.test_uuid = str(uuid.uuid4()) self.name = "%s-%s" % (testclass_name, self.test_uuid) + self.journal_fields = { + 'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid} + + def _initFilter(self, **kwargs): + self.filter = Filter_(self.jail, **kwargs) self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -945,16 +950,20 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=2", "TEST_UUID=%s" % self.test_uuid]) - self.journal_fields = { - 'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid} - self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") - self.filter.start() def tearDown(self): - self.filter.stop() - self.filter.join() # wait for the thread to terminate - pass + if self.filter and self.filter.active: + self.filter.stop() + self.filter.join() # wait for the thread to terminate + pass + + def testJournalFlagsArg(self): + self._initFilter(journalflags=2) # journal.RUNTIME_ONLY + + def __str__(self): + return "MonitorJournalFailures%s(%s)" \ + % (Filter_, hasattr(self, 'name') and self.name or 'tempfile') def assert_correct_ban(self, test_ip, test_attempts): self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react @@ -969,6 +978,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.assertEqual(attempts, test_attempts) def test_grow_file(self): + self._initFilter() + self.filter.start() self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file @@ -998,6 +1009,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.assert_correct_ban("193.168.0.128", 3) def test_delJournalMatch(self): + self._initFilter() + self.filter.start() # Smoke test for removing of match # basic full test diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 6c0f46cf..2fd0efac 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -717,10 +717,7 @@ class Transmitter(TransmitterBase): def testJournalMatch(self): if not filtersystemd: # pragma: no cover - if sys.version_info >= (2, 7): - raise unittest.SkipTest( - "systemd python interface not available") - return + raise unittest.SkipTest("systemd python interface not available") jailName = "TestJail2" self.server.addJail(jailName, "systemd") values = [ @@ -820,10 +817,8 @@ class TransmitterLogging(TransmitterBase): self.setGetTest("logtarget", "STDERR") def testLogTargetSYSLOG(self): - if not os.path.exists("/dev/log") and sys.version_info >= (2, 7): + if not os.path.exists("/dev/log"): raise unittest.SkipTest("'/dev/log' not present") - elif not os.path.exists("/dev/log"): - return self.assertTrue(self.server.getSyslogSocket(), "auto") self.setGetTest("logtarget", "SYSLOG") self.assertTrue(self.server.getSyslogSocket(), "/dev/log")