From cf22fd6b6ba104591ad6e9f517c2e7d3ab301ce9 Mon Sep 17 00:00:00 2001 From: lostcontrol Date: Wed, 13 Sep 2006 21:31:22 +0000 Subject: [PATCH] - Added wildcards support for "logpath" - Added "set addlogpath " and "set dellogpath " - Adapted pyunit test git-svn-id: https://fail2ban.svn.sourceforge.net/svnroot/fail2ban/trunk@354 a942ae1a-1317-0410-a47c-b1dcaea8d605 --- CHANGELOG | 1 + TODO | 3 +- client/beautifier.py | 16 +++-- client/jailreader.py | 9 ++- server/actions.py | 2 - server/filter.py | 126 +++++++++++++++++++++++------------- server/server.py | 8 ++- server/transmitter.py | 9 ++- testcases/filtertestcase.py | 11 ++-- 9 files changed, 120 insertions(+), 65 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index bdb37efb..a4a0241c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,7 @@ Fail2Ban (version 0.7.3) 2006/??/?? ver. 0.7.3 (2006/??/??) - beta ---------- - Added man pages. Thanks to Yaroslav Halchenko +- Added wildcard support for "logpath" ver. 0.7.2 (2006/09/10) - beta ---------- diff --git a/TODO b/TODO index 3a9d37d3..97337c60 100644 --- a/TODO +++ b/TODO @@ -92,10 +92,11 @@ Legend: * remove most of the command lines options if possible. -- add the possibility to specify wildcard in log files. +* add the possibility to specify wildcard in log files. Example: logfile = /var/log/apache2/access-*.log Should we start one thread per file or just one thread per serivce? + We use just one thread per service # autodetect date format in log file. Match the most popular format and sort them using the hit ratio. Should avoid diff --git a/client/beautifier.py b/client/beautifier.py index c2443e8d..916a3f6b 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -50,10 +50,11 @@ class Beautifier: def beautify(self, response): logSys.debug("Beautify " + `response` + " with " + `self.inputCmd`) + inC = self.inputCmd msg = response - if self.inputCmd[0:1] == ['status']: - if len(self.inputCmd) > 1: - msg = "Status for the jail: " + self.inputCmd[1] + "\n" + if inC[0:1] == ['status']: + if len(inC) > 1: + msg = "Status for the jail: " + inC[1] + "\n" msg = msg + "|- " + response[0][0] + "\n" msg = msg + "| |- " + response[0][1][0][0] + ":\t\t" + `response[0][1][0][1]` + "\n" msg = msg + "| `- " + response[0][1][1][0] + ":\t\t" + `response[0][1][1][1]` + "\n" @@ -64,7 +65,7 @@ class Beautifier: msg = "Status\n" msg = msg + "|- " + response[0][0] + ":\t" + `response[0][1]` + "\n" msg = msg + "`- " + response[1][0] + ":\t\t" + response[1][1] - elif self.inputCmd[1:2] == ['loglevel']: + elif inC[1:2] == ['loglevel']: msg = "Current logging level is " if response == 1: msg = msg + "ERROR" @@ -76,9 +77,14 @@ class Beautifier: msg = msg + "DEBUG" else: msg = msg + `response` - elif self.inputCmd == ["stop"]: + elif inC == ["stop"]: if response == None: msg = "Shutdown successful" + elif inC[2] == "logpath" or inC[2] == "addlogpath" or inC[2] == "dellogpath": + msg = "Current monitored log file(s):\n" + for path in response[:-1]: + msg = msg + "|- " + path + "\n" + msg = msg + "`- " + response[len(response)-1] return msg def beautifyError(self, response): diff --git a/client/jailreader.py b/client/jailreader.py index 986f3645..4df4ecc8 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -24,7 +24,8 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, re +import logging, re, glob + from configreader import ConfigReader from filterreader import FilterReader from actionreader import ActionReader @@ -95,7 +96,11 @@ class JailReader(ConfigReader): stream = [["add", self.name]] for opt in self.opts: if opt == "logpath": - stream.append(["set", self.name, "logpath", self.opts[opt]]) + pathList = glob.glob(self.opts[opt]) + if len(pathList) == 0: + logSys.error("No file found for " + self.opts[opt]) + for path in pathList: + stream.append(["set", self.name, "addlogpath", path]) elif opt == "maxretry": stream.append(["set", self.name, "maxretry", self.opts[opt]]) elif opt == "maxtime": diff --git a/server/actions.py b/server/actions.py index 7cf7d692..66a0eecd 100644 --- a/server/actions.py +++ b/server/actions.py @@ -128,7 +128,6 @@ class Actions(JailThread): # @return True if an IP address get banned def checkBan(self): - logSys.debug("Check for IP address to ban") ticket = self.jail.getFailTicket() if ticket != False: aInfo = dict() @@ -148,7 +147,6 @@ class Actions(JailThread): # Unban IP address which are outdated. def checkUnBan(self): - logSys.debug("Check for IP address to unban") for ticket in self.banManager.unBanList(time.time()): aInfo = dict() aInfo["ip"] = ticket.getIP() diff --git a/server/filter.py b/server/filter.py index a4bbe3db..a83a66c7 100644 --- a/server/filter.py +++ b/server/filter.py @@ -57,9 +57,10 @@ class Filter(JailThread): ## The failures manager. self.failManager = FailManager() ## The log file handler. - self.fileHandler = None + self.crtHandler = None + self.crtFilename = None ## The log file path. - self.logPath = '' + self.logPath = [] ## The regular expression matching the failure. self.failRegex = '' self.failRegexObj = None @@ -68,26 +69,49 @@ class Filter(JailThread): ## The ignore IP list. self.ignoreIpList = [] ## The time of the last modification of the file. - self.lastModTime = 0 + self.lastModTime = dict() ## The last position of the file. - self.lastPos = 0 + self.lastPos = dict() ## The last date in tht log file. - self.lastDate = 0 - ## The file statistics. - self.logStats = None + self.lastDate = dict() + self.file404Cnt = dict() + self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() - self.fileNotFoundCnt = 0 logSys.info("Created Filter") ## - # Set the log file path + # Add a log file path # - # @param value log file path + # @param path log file path - def setLogPath(self, value): - self.logPath = value - logSys.info("Set logfile = %s" % value) + def addLogPath(self, path): + try: + self.logPath.index(path) + logSys.error(path + " already exists") + except ValueError: + self.logPath.append(path) + # Initialize default values + self.lastDate[path] = 0 + self.lastModTime[path] = 0 + self.lastPos[path] = 0 + self.file404Cnt[path] = 0 + logSys.info("Added logfile = %s" % path) + + ## + # Delete a log path + # + # @param path the log file to delete + + def delLogPath(self, path): + try: + self.logPath.remove(path) + del self.lastDate[path] + del self.lastModTime[path] + del self.lastPos[path] + logSys.info("Removed logfile = %s" % path) + except ValueError: + logSys.error(path + " is not monitored") ## # Get the log file path @@ -213,11 +237,16 @@ class Filter(JailThread): def run(self): self.setActive(True) + prevModified = False while self.isActive(): if not self.isIdle: - if self.isModified(): - self.getFailures() + if prevModified: self.dateDetector.sortTemplate() + prevModified = False + for file in self.logPath: + if self.isModified(file): + self.getFailures(file) + prevModified = True try: ticket = self.failManager.toBan() self.jail.putFailTicket(ticket) @@ -267,19 +296,21 @@ class Filter(JailThread): ## # Open the log file. - def openLogFile(self): + def openLogFile(self, filename): """ Opens the log file specified on init. """ try: - self.fileHandler = open(self.logPath) + self.crtFilename = filename + self.crtHandler = open(filename) except OSError: - logSys.error("Unable to open "+self.logPath) + logSys.error("Unable to open " + filename) ## # Close the log file. def closeLogFile(self): - self.fileHandler.close() + self.crtFilename = None + self.crtHandler.close() ## # Checks if the log file has been modified. @@ -287,23 +318,23 @@ class Filter(JailThread): # Checks if the log file has been modified using os.stat(). # @return True if log file has been modified - def isModified(self): + def isModified(self, filename): try: - self.logStats = os.stat(self.logPath) - self.fileNotFoundCnt = 0 - if self.lastModTime == self.logStats.st_mtime: + logStats = os.stat(filename) + self.file404Cnt[filename] = 0 + if self.lastModTime[filename] == logStats.st_mtime: return False else: - logSys.debug(self.logPath + " has been modified") - self.lastModTime = self.logStats.st_mtime + logSys.debug(filename + " has been modified") + self.lastModTime[filename] = logStats.st_mtime return True except OSError: - logSys.error("Unable to get stat on " + self.logPath) - self.fileNotFoundCnt = self.fileNotFoundCnt + 1 - if self.fileNotFoundCnt > 2: + logSys.error("Unable to get stat on " + filename) + self.file404Cnt[filename] = self.file404Cnt[filename] + 1 + if self.file404Cnt[filename] > 2: logSys.warn("Too much read error. Set the jail idle") self.jail.setIdle(True) - self.fileNotFoundCnt = 0 + self.file404Cnt[filename] = 0 return False ## @@ -314,22 +345,23 @@ class Filter(JailThread): # timestamp in order to detect this. def setFilePos(self): - line = self.fileHandler.readline() - if self.lastDate < self.dateDetector.getTime(line): - logSys.debug("Date " + `self.lastDate` + " is " + "smaller than " + - `self.dateDetector.getTime(line)`) - logSys.debug("Log rotation detected for " + self.logPath) - self.lastPos = 0 - - logSys.debug("Setting file position to " + `self.lastPos` + " for " + - self.logPath) - self.fileHandler.seek(self.lastPos) + line = self.crtHandler.readline() + lastDate = self.lastDate[self.crtFilename] + lineDate = self.dateDetector.getUnixTime(line) + if lastDate < lineDate: + logSys.debug("Date " + `lastDate` + " is smaller than " + `lineDate`) + logSys.debug("Log rotation detected for " + self.crtFilename) + self.lastPos[self.crtFilename] = 0 + lastPos = self.lastPos[self.crtFilename] + logSys.debug("Setting file position to " + `lastPos` + " for " + + self.crtFilename) + self.crtHandler.seek(lastPos) ## # Get the file position. def getFilePos(self): - return self.fileHandler.tell() + return self.crtHandler.tell() ## # Gets all the failure in the log file. @@ -338,18 +370,20 @@ class Filter(JailThread): # time.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self): + def getFailures(self, filename): ipList = dict() - logSys.debug(self.logPath) - self.openLogFile() + logSys.debug(filename) + self.openLogFile(filename) self.setFilePos() lastLine = None - for line in self.fileHandler: + for line in self.crtHandler: try: # Try to convert UTF-8 string to Latin-1 line = line.decode('utf-8').encode('latin-1') except UnicodeDecodeError: pass + except UnicodeEncodeError: + logSys.warn("Mmhh... Are you sure you watch the correct file?") if not self.dateDetector.matchTime(line): # There is no valid time in this line continue @@ -364,9 +398,9 @@ class Filter(JailThread): continue logSys.debug("Found "+ip) self.failManager.addFailure(FailTicket(ip, unixTime)) - self.lastPos = self.getFilePos() + self.lastPos[filename] = self.getFilePos() if lastLine: - self.lastDate = self.dateDetector.getTime(lastLine) + self.lastDate[filename] = self.dateDetector.getTime(lastLine) self.closeLogFile() ## diff --git a/server/server.py b/server/server.py index 974d04f1..0b39a417 100644 --- a/server/server.py +++ b/server/server.py @@ -123,9 +123,13 @@ class Server: raise ServerUnknownJail(name) # Filter - def setLogPath(self, name, file): + def addLogPath(self, name, file): if self.jails.has_key(name): - self.jails[name].getFilter().setLogPath(file) + self.jails[name].getFilter().addLogPath(file) + + def delLogPath(self, name, file): + if self.jails.has_key(name): + self.jails[name].getFilter().delLogPath(file) def getLogPath(self, name): return self.getFilter(name).getLogPath() diff --git a/server/transmitter.py b/server/transmitter.py index 2415cf73..c01c0e7f 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -129,9 +129,14 @@ class Transmitter: self.server.setIdleJail(name, False) return self.server.getIdleJail(name) # Filter - elif action[1] == "logpath": + elif action[1] == "addlogpath": + value = action[2:] + for path in value: + self.server.addLogPath(name, path) + return self.server.getLogPath(name) + elif action[1] == "dellogpath": value = action[2] - self.server.setLogPath(name, value) + self.server.delLogPath(name, value) return self.server.getLogPath(name) elif action[1] == "timeregex": value = action[2] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index f23cfd10..be62fc5a 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -52,20 +52,21 @@ class IgnoreIP(unittest.TestCase): class LogFile(unittest.TestCase): + filename = "testcases/files/testcase01.log" + def setUp(self): """Call before every test case.""" self.filter = Filter(None) - self.filter.setLogPath("testcases/files/testcase01.log") + self.filter.addLogPath(LogFile.filename) def tearDown(self): """Call after every test case.""" def testOpen(self): - self.filter.openLogFile() + self.filter.openLogFile(LogFile.filename) def testIsModified(self): - self.filter.openLogFile() - self.assertTrue(self.filter.isModified()) + self.assertTrue(self.filter.isModified(LogFile.filename)) class GetFailures(unittest.TestCase): @@ -73,7 +74,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" self.filter = Filter(None) - self.filter.setLogPath("testcases/files/testcase01.log") + self.filter.addLogPath("testcases/files/testcase01.log") self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") self.filter.setTimePattern("%b %d %H:%M:%S") self.filter.setFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) (?:::f{4,6}:)?(?P\S*)")