mirror of
https://github.com/fail2ban/fail2ban.git
synced 2026-03-11 08:55:31 +00:00
RF: Refactor date detector and date template elements
Changes include to use Python class properties, merge some date patterns, and change ISO8601 date template to DatePatternRegex class.
This commit is contained in:
Steven Hiscocks
parent
1e1261ccb4
commit
f2ddb3e3d0
8 changed files with 332 additions and 443 deletions
|
|
@ -24,82 +24,94 @@ __license__ = "GPL"
|
|||
import sys, time, logging
|
||||
from threading import Lock
|
||||
|
||||
from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601
|
||||
from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch
|
||||
|
||||
# Gets the instance of the logger.
|
||||
logSys = logging.getLogger(__name__)
|
||||
|
||||
class DateDetector:
|
||||
|
||||
class DateDetector(object):
|
||||
"""Manages one or more date templates to find a date within a log line.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise the date detector.
|
||||
"""
|
||||
self.__lock = Lock()
|
||||
self.__templates = list()
|
||||
self.__known_names = set()
|
||||
|
||||
def _appendTemplate(self, template):
|
||||
name = template.getName()
|
||||
name = template.name
|
||||
if name in self.__known_names:
|
||||
raise ValueError("There is already a template with name %s" % name)
|
||||
raise ValueError(
|
||||
"There is already a template with name %s" % name)
|
||||
self.__known_names.add(name)
|
||||
self.__templates.append(template)
|
||||
|
||||
|
||||
def appendTemplate(self, template):
|
||||
"""Add a date template to manage and use in search of dates.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
template : DateTemplate or str
|
||||
Can be either a `DateTemplate` instance, or a string which will
|
||||
be used as the pattern for the `DatePatternRegex` template. The
|
||||
template will then be added to the detector.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If a template already exists with the same name.
|
||||
"""
|
||||
if isinstance(template, str):
|
||||
template = DatePatternRegex(template)
|
||||
DateDetector._appendTemplate(self, template)
|
||||
self._appendTemplate(template)
|
||||
|
||||
def addDefaultTemplate(self):
|
||||
"""Add Fail2Ban's default set of date templates.
|
||||
"""
|
||||
self.__lock.acquire()
|
||||
try:
|
||||
# asctime with subsecond: Sun Jan 23 21:59:59.011 2005
|
||||
self.appendTemplate("%a %b %d %H:%M:%S\.%f %Y")
|
||||
# asctime: Sun Jan 23 21:59:59 2005
|
||||
self.appendTemplate("%a %b %d %H:%M:%S %Y")
|
||||
# asctime without year: Sun Jan 23 21:59:59
|
||||
self.appendTemplate("%a %b %d %H:%M:%S")
|
||||
# standard: Jan 23 21:59:59
|
||||
self.appendTemplate("%b %d %H:%M:%S")
|
||||
# proftpd date: 2005-01-23 21:59:59,333
|
||||
self.appendTemplate("%Y-%m-%d %H:%M:%S,%f")
|
||||
# simple date: 2005-01-23 21:59:59
|
||||
self.appendTemplate("%Y-%m-%d %H:%M:%S")
|
||||
# asctime with optional day, subsecond and/or year:
|
||||
# Sun Jan 23 21:59:59.011 2005
|
||||
self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?")
|
||||
# simple date, optional subsecond (proftpd):
|
||||
# 2005-01-23 21:59:59
|
||||
self.appendTemplate("%Y-%m-%d %H:%M:%S(?:,%f)?")
|
||||
# simple date: 2005/01/23 21:59:59
|
||||
self.appendTemplate("%Y/%m/%d %H:%M:%S")
|
||||
# simple date too (from x11vnc): 23/01/2005 21:59:59
|
||||
self.appendTemplate("%d/%m/%Y %H:%M:%S")
|
||||
# previous one but with year given by 2 digits: 23/01/05 21:59:59
|
||||
# and with optional year given by 2 digits: 23/01/05 21:59:59
|
||||
# (See http://bugs.debian.org/537610)
|
||||
self.appendTemplate("%d/%m/%y %H:%M:%S")
|
||||
# Apache format [31/Oct/2006:09:22:55 -0000]
|
||||
self.appendTemplate("%d/%b/%Y:%H:%M:%S %z")
|
||||
# [31/Oct/2006:09:22:55]
|
||||
self.appendTemplate("%d/%b/%Y:%H:%M:%S")
|
||||
self.appendTemplate("%d/%m/(?:%Y|%y) %H:%M:%S")
|
||||
# Apache format optional time zone:
|
||||
# [31/Oct/2006:09:22:55 -0000]
|
||||
self.appendTemplate("%d/%b/%Y:%H:%M:%S(?: %z)?")
|
||||
# CPanel 05/20/2008:01:57:39
|
||||
self.appendTemplate("%m/%d/%Y:%H:%M:%S")
|
||||
# custom for syslog-ng 2006.12.21 06:43:20
|
||||
self.appendTemplate("%Y\.%m\.%d %H:%M:%S")
|
||||
# named 26-Jul-2007 15:20:52.252
|
||||
self.appendTemplate("%d-%b-%Y %H:%M:%S\.%f")
|
||||
# roundcube 26-Jul-2007 15:20:52 +0200
|
||||
self.appendTemplate("%d-%b-%Y %H:%M:%S %z")
|
||||
self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?")
|
||||
# 26-Jul-2007 15:20:52
|
||||
self.appendTemplate("%d-%b-%Y %H:%M:%S")
|
||||
# 17-07-2008 17:23:25
|
||||
self.appendTemplate("%d-%m-%Y %H:%M:%S")
|
||||
# 01-27-2012 16:22:44.252
|
||||
# subseconds explicit to avoid possible %m<->%d confusion
|
||||
# with previous
|
||||
self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f")
|
||||
# TAI64N
|
||||
template = DateTai64n()
|
||||
template.setName("TAI64N")
|
||||
template.name = "TAI64N"
|
||||
self.appendTemplate(template)
|
||||
# Epoch
|
||||
template = DateEpoch()
|
||||
template.setName("Epoch")
|
||||
template.name = "Epoch"
|
||||
self.appendTemplate(template)
|
||||
# ISO 8601
|
||||
template = DateISO8601()
|
||||
template.setName("ISO 8601")
|
||||
self.appendTemplate(template)
|
||||
self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?")
|
||||
# Only time information in the log
|
||||
self.appendTemplate("^%H:%M:%S")
|
||||
# <09/16/08@05:03:30>
|
||||
|
|
@ -112,25 +124,60 @@ class DateDetector:
|
|||
self.appendTemplate("^%b-%d-%y %H:%M:%S")
|
||||
finally:
|
||||
self.__lock.release()
|
||||
|
||||
def getTemplates(self):
|
||||
|
||||
@property
|
||||
def templates(self):
|
||||
"""List of template instances managed by the detector.
|
||||
"""
|
||||
return self.__templates
|
||||
|
||||
def matchTime(self, line, incHits=True):
|
||||
|
||||
def matchTime(self, line):
|
||||
"""Attempts to find date on a log line using templates.
|
||||
|
||||
This uses the templates' `matchDate` method in an attempt to find
|
||||
a date. It also increments the match hit count for the winning
|
||||
template.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Line which is searched by the date templates.
|
||||
|
||||
Returns
|
||||
-------
|
||||
re.MatchObject
|
||||
The regex match returned from the first successfully matched
|
||||
template.
|
||||
"""
|
||||
self.__lock.acquire()
|
||||
try:
|
||||
for template in self.__templates:
|
||||
match = template.matchDate(line)
|
||||
if not match is None:
|
||||
logSys.debug("Matched time template %s" % template.getName())
|
||||
if incHits:
|
||||
template.incHits()
|
||||
logSys.debug("Matched time template %s" % template.name)
|
||||
template.hits += 1
|
||||
return match
|
||||
return None
|
||||
finally:
|
||||
self.__lock.release()
|
||||
|
||||
def getTime(self, line):
|
||||
"""Attempts to return the date on a log line using templates.
|
||||
|
||||
This uses the templates' `getDate` method in an attempt to find
|
||||
a date.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Line which is searched by the date templates.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
The Unix timestamp returned from the first successfully matched
|
||||
template.
|
||||
"""
|
||||
self.__lock.acquire()
|
||||
try:
|
||||
for template in self.__templates:
|
||||
|
|
@ -138,7 +185,8 @@ class DateDetector:
|
|||
date = template.getDate(line)
|
||||
if date is None:
|
||||
continue
|
||||
logSys.debug("Got time %f for \"%r\" using template %s" % (date[0], date[1].group(), template.getName()))
|
||||
logSys.debug("Got time %f for \"%r\" using template %s" %
|
||||
(date[0], date[1].group(), template.name))
|
||||
return date
|
||||
except ValueError:
|
||||
pass
|
||||
|
|
@ -146,16 +194,19 @@ class DateDetector:
|
|||
finally:
|
||||
self.__lock.release()
|
||||
|
||||
##
|
||||
# Sort the template lists using the hits score. This method is not called
|
||||
# in this object and thus should be called from time to time.
|
||||
|
||||
def sortTemplate(self):
|
||||
"""Sort the date templates by number of hits
|
||||
|
||||
Sort the template lists using the hits score. This method is not
|
||||
called in this object and thus should be called from time to time.
|
||||
This ensures the most commonly matched templates are checked first,
|
||||
improving performance of matchTime and getTime.
|
||||
"""
|
||||
self.__lock.acquire()
|
||||
try:
|
||||
logSys.debug("Sorting the template list")
|
||||
self.__templates.sort(key=lambda x: x.getHits(), reverse=True)
|
||||
self.__templates.sort(key=lambda x: x.hits, reverse=True)
|
||||
t = self.__templates[0]
|
||||
logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits()))
|
||||
logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits))
|
||||
finally:
|
||||
self.__lock.release()
|
||||
|
|
|
|||
|
|
@ -26,211 +26,132 @@ __license__ = "GPL"
|
|||
|
||||
import re, time, calendar
|
||||
import logging
|
||||
from abc import abstractmethod
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
|
||||
from .mytime import MyTime
|
||||
from . import iso8601
|
||||
from .strptime import reGroupDictStrptime, timeRE
|
||||
|
||||
logSys = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DateTemplate(object):
|
||||
|
||||
"""A template which searches for and returns a date from a log line.
|
||||
|
||||
This is an not functional abstract class which other templates should
|
||||
inherit from.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.__name = ""
|
||||
self.__regex = ""
|
||||
self.__cRegex = None
|
||||
self.__hits = 0
|
||||
|
||||
def setName(self, name):
|
||||
self.__name = name
|
||||
|
||||
def getName(self):
|
||||
return self.__name
|
||||
|
||||
"""Initialise the date template.
|
||||
"""
|
||||
self._name = ""
|
||||
self._regex = ""
|
||||
self._cRegex = None
|
||||
self.hits = 0
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
"""Name assigned to template.
|
||||
"""
|
||||
return self._name
|
||||
|
||||
@name.setter
|
||||
def name(self, name):
|
||||
self._name = name
|
||||
|
||||
def getRegex(self):
|
||||
return self._regex
|
||||
|
||||
def setRegex(self, regex, wordBegin=True):
|
||||
#logSys.debug(u"setRegex for %s is %r" % (self.__name, regex))
|
||||
"""Sets regex to use for searching for date in log line.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
regex : str
|
||||
The regex the template will use for searching for a date.
|
||||
wordBegin : bool
|
||||
Defines whether the regex should be modified to search at
|
||||
begining of a word, by adding "\\b" to start of regex.
|
||||
Default True.
|
||||
|
||||
Raises
|
||||
------
|
||||
re.error
|
||||
If regular expression fails to compile
|
||||
"""
|
||||
regex = regex.strip()
|
||||
if (wordBegin and not re.search(r'^\^', regex)):
|
||||
regex = r'\b' + regex
|
||||
self.__regex = regex
|
||||
self.__cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE)
|
||||
|
||||
def getRegex(self):
|
||||
return self.__regex
|
||||
|
||||
def getHits(self):
|
||||
return self.__hits
|
||||
self._regex = regex
|
||||
self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE)
|
||||
|
||||
def incHits(self):
|
||||
self.__hits += 1
|
||||
regex = property(getRegex, setRegex, doc=
|
||||
"""Regex used to search for date.
|
||||
""")
|
||||
|
||||
def resetHits(self):
|
||||
self.__hits = 0
|
||||
|
||||
def matchDate(self, line):
|
||||
dateMatch = self.__cRegex.search(line)
|
||||
"""Check if regex for date matches on a log line.
|
||||
"""
|
||||
dateMatch = self._cRegex.search(line)
|
||||
return dateMatch
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def getDate(self, line):
|
||||
raise Exception("matchDate() is abstract")
|
||||
"""Abstract method, which should return the date for a log line
|
||||
|
||||
This should return the date for a log line, typically taking the
|
||||
date from the part of the line which matched the templates regex.
|
||||
This requires abstraction, therefore just raises exception.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Log line, of which the date should be extracted from.
|
||||
|
||||
Raises
|
||||
------
|
||||
NotImplementedError
|
||||
Abstract method, therefore always returns this.
|
||||
"""
|
||||
raise NotImplementedError("getDate() is abstract")
|
||||
|
||||
|
||||
class DateEpoch(DateTemplate):
|
||||
|
||||
"""A date template which searches for Unix timestamps.
|
||||
|
||||
This includes Unix timestamps which appear at start of a line, optionally
|
||||
within square braces (nsd), or on SELinux audit log lines.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise the date template.
|
||||
"""
|
||||
DateTemplate.__init__(self)
|
||||
self.setRegex("(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))")
|
||||
|
||||
self.regex = "(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))"
|
||||
|
||||
def getDate(self, line):
|
||||
"""Method to return the date for a log line.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Log line, of which the date should be extracted from.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(float, str)
|
||||
Tuple containing a Unix timestamp, and the string of the date
|
||||
which was matched and in turned used to calculated the timestamp.
|
||||
"""
|
||||
dateMatch = self.matchDate(line)
|
||||
if dateMatch:
|
||||
# extract part of format which represents seconds since epoch
|
||||
return (float(dateMatch.group()), dateMatch)
|
||||
return None
|
||||
|
||||
|
||||
##
|
||||
# Use strptime() to parse a date. Our current locale is the 'C'
|
||||
# one because we do not set the locale explicitly. This is POSIX
|
||||
# standard.
|
||||
|
||||
class DateStrptime(DateTemplate):
|
||||
|
||||
TABLE = dict()
|
||||
TABLE["Jan"] = ["Sty"]
|
||||
TABLE["Feb"] = [u"Fév", "Lut"]
|
||||
TABLE["Mar"] = [u"Mär", "Mar"]
|
||||
TABLE["Apr"] = ["Avr", "Kwi"]
|
||||
TABLE["May"] = ["Mai", "Maj"]
|
||||
TABLE["Jun"] = ["Lip"]
|
||||
TABLE["Jul"] = ["Sie"]
|
||||
TABLE["Aug"] = ["Aou", "Wrz"]
|
||||
TABLE["Sep"] = ["Sie"]
|
||||
TABLE["Oct"] = [u"Paź"]
|
||||
TABLE["Nov"] = ["Lis"]
|
||||
TABLE["Dec"] = [u"Déc", "Dez", "Gru"]
|
||||
|
||||
def __init__(self):
|
||||
DateTemplate.__init__(self)
|
||||
self._pattern = ""
|
||||
self._unsupportedStrptimeBits = False
|
||||
|
||||
def setPattern(self, pattern):
|
||||
self._unsupported_f = not DateStrptime._f and re.search('%f', pattern)
|
||||
self._unsupported_z = not DateStrptime._z and re.search('%z', pattern)
|
||||
self._pattern = pattern
|
||||
|
||||
def getPattern(self):
|
||||
return self._pattern
|
||||
|
||||
#@staticmethod
|
||||
def convertLocale(date):
|
||||
for t in DateStrptime.TABLE:
|
||||
for m in DateStrptime.TABLE[t]:
|
||||
if date.find(m) >= 0:
|
||||
logSys.debug(u"Replacing %r with %r in %r" %
|
||||
(m, t, date))
|
||||
return date.replace(m, t)
|
||||
return date
|
||||
convertLocale = staticmethod(convertLocale)
|
||||
|
||||
def getDate(self, line):
|
||||
dateMatch = self.matchDate(line)
|
||||
|
||||
if dateMatch:
|
||||
datePattern = self.getPattern()
|
||||
if self._unsupported_f:
|
||||
if dateMatch.group('_f'):
|
||||
datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern)
|
||||
logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern))
|
||||
if self._unsupported_z:
|
||||
if dateMatch.group('_z'):
|
||||
datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern)
|
||||
logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern))
|
||||
try:
|
||||
# Try first with 'C' locale
|
||||
date = datetime.strptime(dateMatch.group(), datePattern)
|
||||
except ValueError:
|
||||
# Try to convert date string to 'C' locale
|
||||
conv = self.convertLocale(dateMatch.group())
|
||||
try:
|
||||
date = datetime.strptime(conv, self.getPattern())
|
||||
except (ValueError, re.error), e:
|
||||
# Try to add the current year to the pattern. Should fix
|
||||
# the "Feb 29" issue.
|
||||
opattern = self.getPattern()
|
||||
# makes sense only if %Y is not in already:
|
||||
if not '%Y' in opattern:
|
||||
pattern = "%s %%Y" % opattern
|
||||
conv += " %s" % MyTime.gmtime()[0]
|
||||
date = datetime.strptime(conv, pattern)
|
||||
else:
|
||||
# we are helpless here
|
||||
raise ValueError(
|
||||
"Given pattern %r does not match. Original "
|
||||
"exception was %r and Feb 29 workaround could not "
|
||||
"be tested due to already present year mark in the "
|
||||
"pattern" % (opattern, e))
|
||||
|
||||
if self._unsupported_z:
|
||||
z = dateMatch.group('_z')
|
||||
if z:
|
||||
delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:]))
|
||||
direction = z[0]
|
||||
logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta))
|
||||
# here we reverse the effect of the timezone and force it to UTC
|
||||
if direction == '+':
|
||||
date -= delta
|
||||
else:
|
||||
date += delta
|
||||
date = date.replace(tzinfo=iso8601.Utc())
|
||||
else:
|
||||
logSys.warning("No _z group captured and %%z is not supported on current platform"
|
||||
" - timezone ignored and assumed to be localtime. date: %s on line: %s"
|
||||
% (date, line))
|
||||
|
||||
if date.year < 2000:
|
||||
# There is probably no year field in the logs
|
||||
# NOTE: Possibly makes week/year day incorrect
|
||||
date = date.replace(year=MyTime.gmtime()[0])
|
||||
# Bug fix for #1241756
|
||||
# If the date is greater than the current time, we suppose
|
||||
# that the log is not from this year but from the year before
|
||||
if date > MyTime.now():
|
||||
logSys.debug(
|
||||
u"Correcting deduced year by one since %s > now (%s)" %
|
||||
(date, MyTime.time()))
|
||||
date = date.replace(year=date.year-1)
|
||||
elif date.month == 1 and date.day == 1:
|
||||
# If it is Jan 1st, it is either really Jan 1st or there
|
||||
# is neither month nor day in the log.
|
||||
# NOTE: Possibly makes week/year day incorrect
|
||||
date = date.replace(
|
||||
month=MyTime.gmtime()[1], day=MyTime.gmtime()[2])
|
||||
|
||||
if date.tzinfo:
|
||||
return ( calendar.timegm(date.utctimetuple()), dateMatch )
|
||||
else:
|
||||
return ( time.mktime(date.utctimetuple()), dateMatch )
|
||||
|
||||
return None
|
||||
|
||||
try:
|
||||
time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f")
|
||||
DateStrptime._f = True
|
||||
except (ValueError, KeyError):
|
||||
DateTemplate._f = False
|
||||
|
||||
try:
|
||||
time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z")
|
||||
DateStrptime._z = True
|
||||
except ValueError:
|
||||
DateStrptime._z = False
|
||||
|
||||
class DatePatternRegex(DateStrptime):
|
||||
class DatePatternRegex(DateTemplate):
|
||||
_patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys())
|
||||
_patternName = {
|
||||
'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
|
||||
|
|
@ -241,39 +162,97 @@ class DatePatternRegex(DateStrptime):
|
|||
for key in set(timeRE) - set(_patternName): # may not have them all...
|
||||
_patternName[key] = "%%%s" % key
|
||||
|
||||
def __init__(self, pattern=None, **kwargs):
|
||||
super(DatePatternRegex, self).__init__()
|
||||
if pattern:
|
||||
self.setPattern(pattern, **kwargs)
|
||||
def __init__(self, pattern=None):
|
||||
"""Initialise date template, with optional regex/pattern
|
||||
|
||||
def setPattern(self, pattern):
|
||||
super(DatePatternRegex, self).setPattern(pattern)
|
||||
super(DatePatternRegex, self).setName(
|
||||
re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName)
|
||||
Parameters
|
||||
----------
|
||||
pattern : str
|
||||
Sets the date templates pattern.
|
||||
"""
|
||||
super(DatePatternRegex, self).__init__()
|
||||
self._pattern = None
|
||||
if pattern is not None:
|
||||
self.pattern = pattern
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
"""The pattern used for regex with strptime "%" time fields.
|
||||
|
||||
This should be a valid regular expression, of which matching string
|
||||
will be extracted from the log line. strptime style "%" fields will
|
||||
be replaced by appropriate regular expressions, or custom regex
|
||||
groups with names as per the strptime fields can also be used
|
||||
instead.
|
||||
"""
|
||||
return self._pattern
|
||||
|
||||
@pattern.setter
|
||||
def pattern(self, pattern):
|
||||
self._pattern = pattern
|
||||
self._name = re.sub(
|
||||
self._patternRE, r'%(\1)s', pattern) % self._patternName
|
||||
super(DatePatternRegex, self).setRegex(
|
||||
re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE)
|
||||
|
||||
def getDate(self, line):
|
||||
dateMatch = self.matchDate(line)
|
||||
if dateMatch:
|
||||
return reGroupDictStrptime(dateMatch.groupdict()), dateMatch
|
||||
|
||||
def setRegex(self, line):
|
||||
def setRegex(self, value):
|
||||
raise NotImplementedError("Regex derived from pattern")
|
||||
|
||||
def setName(self, line):
|
||||
@DateTemplate.name.setter
|
||||
def name(self, value):
|
||||
raise NotImplementedError("Name derived from pattern")
|
||||
|
||||
def getDate(self, line):
|
||||
"""Method to return the date for a log line.
|
||||
|
||||
This uses a custom version of strptime, using the named groups
|
||||
from the instances `pattern` property.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Log line, of which the date should be extracted from.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(float, str)
|
||||
Tuple containing a Unix timestamp, and the string of the date
|
||||
which was matched and in turned used to calculated the timestamp.
|
||||
"""
|
||||
dateMatch = self.matchDate(line)
|
||||
if dateMatch:
|
||||
groupdict = dict(
|
||||
(key, value)
|
||||
for key, value in dateMatch.groupdict().iteritems()
|
||||
if value is not None)
|
||||
return reGroupDictStrptime(groupdict), dateMatch
|
||||
|
||||
class DateTai64n(DateTemplate):
|
||||
|
||||
"""A date template which matches TAI64N formate timestamps.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise the date template.
|
||||
"""
|
||||
DateTemplate.__init__(self)
|
||||
# We already know the format for TAI64N
|
||||
# yoh: we should not add an additional front anchor
|
||||
self.setRegex("@[0-9a-f]{24}", wordBegin=False)
|
||||
|
||||
|
||||
def getDate(self, line):
|
||||
"""Method to return the date for a log line.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
line : str
|
||||
Log line, of which the date should be extracted from.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(float, str)
|
||||
Tuple containing a Unix timestamp, and the string of the date
|
||||
which was matched and in turned used to calculated the timestamp.
|
||||
"""
|
||||
dateMatch = self.matchDate(line)
|
||||
if dateMatch:
|
||||
# extract part of format which represents seconds since epoch
|
||||
|
|
@ -282,19 +261,3 @@ class DateTai64n(DateTemplate):
|
|||
# convert seconds from HEX into local time stamp
|
||||
return (int(seconds_since_epoch, 16), dateMatch)
|
||||
return None
|
||||
|
||||
|
||||
class DateISO8601(DateTemplate):
|
||||
|
||||
def __init__(self):
|
||||
DateTemplate.__init__(self)
|
||||
self.setRegex(iso8601.ISO8601_REGEX_RAW)
|
||||
|
||||
def getDate(self, line):
|
||||
dateMatch = self.matchDate(line)
|
||||
if dateMatch:
|
||||
# Parses the date.
|
||||
value = dateMatch.group()
|
||||
return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch)
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from .failmanager import FailManagerEmpty, FailManager
|
|||
from .ticket import FailTicket
|
||||
from .jailthread import JailThread
|
||||
from .datedetector import DateDetector
|
||||
from .datetemplate import DatePatternRegex, DateISO8601, DateEpoch, DateTai64n
|
||||
from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n
|
||||
from .mytime import MyTime
|
||||
from .failregex import FailRegex, Regex, RegexException
|
||||
from .action import CommandAction
|
||||
|
|
@ -202,24 +202,20 @@ class Filter(JailThread):
|
|||
if pattern is None:
|
||||
self.dateDetector = None
|
||||
return
|
||||
elif pattern.upper() == "ISO8601":
|
||||
template = DateISO8601()
|
||||
template.setName("ISO8601")
|
||||
elif pattern.upper() == "EPOCH":
|
||||
template = DateEpoch()
|
||||
template.setName("Epoch")
|
||||
template.name = "Epoch"
|
||||
elif pattern.upper() == "TAI64N":
|
||||
template = DateTai64n()
|
||||
template.setName("TAI64N")
|
||||
template.name = "TAI64N"
|
||||
else:
|
||||
template = DatePatternRegex()
|
||||
template.setPattern(pattern)
|
||||
template = DatePatternRegex(pattern)
|
||||
self.dateDetector = DateDetector()
|
||||
self.dateDetector.appendTemplate(template)
|
||||
logSys.info("Date pattern set to `%r`: `%s`" %
|
||||
(pattern, template.getName()))
|
||||
(pattern, template.name))
|
||||
logSys.debug("Date pattern regex for %r: %s" %
|
||||
(pattern, template.getRegex()))
|
||||
(pattern, template.regex))
|
||||
|
||||
##
|
||||
# Get the date detector pattern, or Default Detectors if not changed
|
||||
|
|
@ -228,15 +224,15 @@ class Filter(JailThread):
|
|||
|
||||
def getDatePattern(self):
|
||||
if self.dateDetector is not None:
|
||||
templates = self.dateDetector.getTemplates()
|
||||
templates = self.dateDetector.templates
|
||||
if len(templates) > 1:
|
||||
return None, "Default Detectors"
|
||||
elif len(templates) == 1:
|
||||
if hasattr(templates[0], "getPattern"):
|
||||
pattern = templates[0].getPattern()
|
||||
if hasattr(templates[0], "pattern"):
|
||||
pattern = templates[0].pattern
|
||||
else:
|
||||
pattern = None
|
||||
return pattern, templates[0].getName()
|
||||
return pattern, templates[0].name
|
||||
|
||||
##
|
||||
# Set the maximum retry value.
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
|
||||
# vi: set ft=python sts=4 ts=4 sw=4 et:
|
||||
|
||||
# Copyright (c) 2007 Michael Twomey
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the
|
||||
# "Software"), to deal in the Software without restriction, including
|
||||
# without limitation the rights to use, copy, modify, merge, publish,
|
||||
# distribute, sublicense, and/or sell copies of the Software, and to
|
||||
# permit persons to whom the Software is furnished to do so, subject to
|
||||
# the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included
|
||||
# in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
"""ISO 8601 date time string parsing
|
||||
|
||||
Basic usage:
|
||||
>>> import iso8601
|
||||
>>> iso8601.parse_date("2007-01-25T12:00:00Z")
|
||||
datetime.datetime(2007, 1, 25, 12, 0, tzinfo=<iso8601.iso8601.Utc ...>)
|
||||
>>>
|
||||
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, tzinfo
|
||||
import time
|
||||
import re
|
||||
|
||||
__all__ = ["parse_date", "ParseError"]
|
||||
|
||||
# Adapted from http://delete.me.uk/2005/03/iso8601.html
|
||||
ISO8601_REGEX_RAW = "(?P<year>[0-9]{4})-(?P<month>[0-9]{1,2})-(?P<day>[0-9]{1,2})" \
|
||||
"T(?P<hour>[0-9]{2}):(?P<minute>[0-9]{2})(:(?P<second>[0-9]{2})(\.(?P<fraction>[0-9]+))?)?" \
|
||||
"(?P<timezone>Z|[-+][0-9]{2}(:?[0-9]{2})?)?"
|
||||
ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW)
|
||||
TIMEZONE_REGEX = re.compile("(?P<prefix>[+-])(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})?")
|
||||
|
||||
class ParseError(Exception):
|
||||
"""Raised when there is a problem parsing a date string"""
|
||||
|
||||
# Yoinked from python docs
|
||||
ZERO = timedelta(0)
|
||||
class Utc(tzinfo):
|
||||
"""UTC
|
||||
|
||||
"""
|
||||
def utcoffset(self, dt):
|
||||
return ZERO
|
||||
|
||||
def tzname(self, dt):
|
||||
return "UTC"
|
||||
|
||||
def dst(self, dt):
|
||||
return ZERO
|
||||
UTC = Utc()
|
||||
|
||||
class FixedOffset(tzinfo):
|
||||
"""Fixed offset in hours and minutes from UTC
|
||||
|
||||
"""
|
||||
def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0):
|
||||
self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds)
|
||||
self.__name = name
|
||||
|
||||
def utcoffset(self, dt):
|
||||
return self.__offset
|
||||
|
||||
def tzname(self, dt):
|
||||
return self.__name
|
||||
|
||||
def dst(self, dt):
|
||||
return ZERO
|
||||
|
||||
def __repr__(self):
|
||||
return "<FixedOffset %r>" % self.__name
|
||||
|
||||
def parse_timezone(tzstring):
|
||||
"""Parses ISO 8601 time zone specs into tzinfo offsets
|
||||
|
||||
"""
|
||||
if tzstring == "Z":
|
||||
return UTC
|
||||
|
||||
if tzstring is None:
|
||||
zone_sec = -time.timezone
|
||||
return FixedOffset(name=time.tzname[0],offset_hours=(zone_sec / 3600), offset_minutes=(zone_sec % 3600)/60, offset_seconds=zone_sec % 60)
|
||||
|
||||
m = TIMEZONE_REGEX.match(tzstring)
|
||||
prefix, hours, minutes = m.groups()
|
||||
if minutes is None:
|
||||
minutes = 0
|
||||
else:
|
||||
minutes = int(minutes)
|
||||
hours = int(hours)
|
||||
if prefix == "-":
|
||||
hours = -hours
|
||||
minutes = -minutes
|
||||
return FixedOffset(tzstring, hours, minutes)
|
||||
|
||||
def parse_date(datestring):
|
||||
"""Parses ISO 8601 dates into datetime objects
|
||||
|
||||
The timezone is parsed from the date string. However it is quite common to
|
||||
have dates without a timezone (not strictly correct). In this case the
|
||||
default timezone specified in default_timezone is used. This is UTC by
|
||||
default.
|
||||
"""
|
||||
if not isinstance(datestring, basestring):
|
||||
raise ValueError("Expecting a string %r" % datestring)
|
||||
m = ISO8601_REGEX.match(datestring)
|
||||
if not m:
|
||||
raise ParseError("Unable to parse date string %r" % datestring)
|
||||
groups = m.groupdict()
|
||||
tz = parse_timezone(groups["timezone"])
|
||||
if groups["fraction"] is None:
|
||||
groups["fraction"] = 0
|
||||
else:
|
||||
groups["fraction"] = int(float("0.%s" % groups["fraction"]) * 1e6)
|
||||
|
||||
try:
|
||||
return datetime(int(groups["year"]), int(groups["month"]), int(groups["day"]),
|
||||
int(groups["hour"]), int(groups["minute"]), int(groups["second"]),
|
||||
int(groups["fraction"]), tz)
|
||||
except Exception, e:
|
||||
raise ParseError("Failed to create a valid datetime record due to: %s"
|
||||
% e)
|
||||
|
|
@ -26,11 +26,24 @@ from .mytime import MyTime
|
|||
|
||||
locale_time = LocaleTime()
|
||||
timeRE = TimeRE()
|
||||
if 'z' not in timeRE: # python2.6 not present
|
||||
timeRE['z'] = r"(?P<z>[+-]\d{2}[0-5]\d)"
|
||||
timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
|
||||
|
||||
def reGroupDictStrptime(found_dict):
|
||||
"""This is tweaked from python built-in _strptime"""
|
||||
"""Return time from dictionary of strptime fields
|
||||
|
||||
This is tweaked from python built-in _strptime.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
found_dict : dict
|
||||
Dictionary where keys represent the strptime fields, and values the
|
||||
respective value.
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Unix time stamp.
|
||||
"""
|
||||
|
||||
now = MyTime.now()
|
||||
year = month = day = hour = minute = None
|
||||
|
|
@ -119,9 +132,14 @@ def reGroupDictStrptime(found_dict):
|
|||
week_of_year_start = 0
|
||||
elif group_key == 'z':
|
||||
z = found_dict['z']
|
||||
tzoffset = int(z[1:3]) * 60 + int(z[3:5])
|
||||
if z.startswith("-"):
|
||||
tzoffset = -tzoffset
|
||||
if z == "Z":
|
||||
tzoffset = 0
|
||||
else:
|
||||
tzoffset = int(z[1:3]) * 60 # Hours...
|
||||
if len(z)>3:
|
||||
tzoffset += int(z[-2:]) # ...and minutes
|
||||
if z.startswith("-"):
|
||||
tzoffset = -tzoffset
|
||||
elif group_key == 'Z':
|
||||
# Since -1 is default value only need to worry about setting tz if
|
||||
# it can be something other than -1.
|
||||
|
|
@ -158,7 +176,6 @@ def reGroupDictStrptime(found_dict):
|
|||
month = datetime_result.month
|
||||
day = datetime_result.day
|
||||
# Add timezone info
|
||||
tzname = found_dict.get("Z")
|
||||
if tzoffset is not None:
|
||||
gmtoff = tzoffset * 60
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ __license__ = "GPL"
|
|||
import unittest, calendar, time, datetime, re, pprint
|
||||
from ..server.datedetector import DateDetector
|
||||
from ..server.datetemplate import DateTemplate
|
||||
from ..server.iso8601 import Utc
|
||||
from .utils import setUpMyTime, tearDownMyTime
|
||||
|
||||
class DateDetectorTest(unittest.TestCase):
|
||||
|
|
@ -88,8 +87,9 @@ class DateDetectorTest(unittest.TestCase):
|
|||
(False, "23-01-2005 21:59:59"),
|
||||
(False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break
|
||||
(False, "@4000000041f4104f00000000"), # TAI64N
|
||||
(False, "2005-01-23T20:59:59.252Z"), #ISO 8601
|
||||
(False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC)
|
||||
(False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ
|
||||
(False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local
|
||||
(True, "<01/23/05@21:59:59>"),
|
||||
(True, "050123 21:59:59"), # MySQL
|
||||
(True, "Jan-23-05 21:59:59"), # ASSP like
|
||||
|
|
@ -116,15 +116,15 @@ class DateDetectorTest(unittest.TestCase):
|
|||
self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime))
|
||||
|
||||
def testStableSortTemplate(self):
|
||||
old_names = [x.getName() for x in self.__datedetector.getTemplates()]
|
||||
old_names = [x.name for x in self.__datedetector.templates]
|
||||
self.__datedetector.sortTemplate()
|
||||
# If there were no hits -- sorting should not change the order
|
||||
for old_name, n in zip(old_names, self.__datedetector.getTemplates()):
|
||||
self.assertEqual(old_name, n.getName()) # "Sort must be stable"
|
||||
for old_name, n in zip(old_names, self.__datedetector.templates):
|
||||
self.assertEqual(old_name, n.name) # "Sort must be stable"
|
||||
|
||||
def testAllUniqueTemplateNames(self):
|
||||
self.assertRaises(ValueError, self.__datedetector.appendTemplate,
|
||||
self.__datedetector.getTemplates()[0])
|
||||
self.__datedetector.templates[0])
|
||||
|
||||
def testFullYearMatch_gh130(self):
|
||||
# see https://github.com/fail2ban/fail2ban/pull/130
|
||||
|
|
|
|||
|
|
@ -170,46 +170,46 @@ class TestsUtilsTest(unittest.TestCase):
|
|||
self.assertTrue(pindex > 10) # we should have some traceback
|
||||
self.assertEqual(s[:pindex], s[pindex+1:pindex*2 + 1])
|
||||
|
||||
from ..server import iso8601
|
||||
import datetime
|
||||
import time
|
||||
|
||||
from ..server.datetemplate import DatePatternRegex
|
||||
|
||||
iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
|
||||
|
||||
class CustomDateFormatsTest(unittest.TestCase):
|
||||
|
||||
def testIso8601(self):
|
||||
date = iso8601.parse_date("2007-01-25T12:00:00Z")
|
||||
date = datetime.datetime.fromtimestamp(
|
||||
iso8601.getDate("2007-01-25T12:00:00Z")[0])
|
||||
self.assertEqual(
|
||||
date,
|
||||
datetime.datetime(2007, 1, 25, 12, 0, tzinfo=iso8601.Utc()))
|
||||
self.assertRaises(ValueError, iso8601.parse_date, None)
|
||||
self.assertRaises(ValueError, iso8601.parse_date, date)
|
||||
datetime.datetime(2007, 1, 25, 12, 0))
|
||||
self.assertRaises(TypeError, iso8601.getDate, None)
|
||||
self.assertRaises(TypeError, iso8601.getDate, date)
|
||||
|
||||
self.assertRaises(iso8601.ParseError, iso8601.parse_date, "")
|
||||
self.assertRaises(iso8601.ParseError, iso8601.parse_date, "Z")
|
||||
self.assertEqual(iso8601.getDate(""), None)
|
||||
self.assertEqual(iso8601.getDate("Z"), None)
|
||||
|
||||
self.assertRaises(iso8601.ParseError,
|
||||
iso8601.parse_date, "2007-01-01T120:00:00Z")
|
||||
self.assertRaises(iso8601.ParseError,
|
||||
iso8601.parse_date, "2007-13-01T12:00:00Z")
|
||||
date = iso8601.parse_date("2007-01-25T12:00:00+0400")
|
||||
self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None)
|
||||
self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None)
|
||||
date = datetime.datetime.fromtimestamp(
|
||||
iso8601.getDate("2007-01-25T12:00:00+0400")[0])
|
||||
self.assertEqual(
|
||||
date,
|
||||
datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc()))
|
||||
date = iso8601.parse_date("2007-01-25T12:00:00+04:00")
|
||||
datetime.datetime(2007, 1, 25, 8, 0))
|
||||
date = datetime.datetime.fromtimestamp(
|
||||
iso8601.getDate("2007-01-25T12:00:00+04:00")[0])
|
||||
self.assertEqual(
|
||||
date,
|
||||
datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc()))
|
||||
date = iso8601.parse_date("2007-01-25T12:00:00-0400")
|
||||
datetime.datetime(2007, 1, 25, 8, 0))
|
||||
date = datetime.datetime.fromtimestamp(
|
||||
iso8601.getDate("2007-01-25T12:00:00-0400")[0])
|
||||
self.assertEqual(
|
||||
date,
|
||||
datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc()))
|
||||
date = iso8601.parse_date("2007-01-25T12:00:00-04")
|
||||
datetime.datetime(2007, 1, 25, 16, 0))
|
||||
date = datetime.datetime.fromtimestamp(
|
||||
iso8601.getDate("2007-01-25T12:00:00-04")[0])
|
||||
self.assertEqual(
|
||||
date,
|
||||
datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc()))
|
||||
|
||||
def testTimeZone(self):
|
||||
# Just verify consistent operation and improve coverage ;)
|
||||
self.assertEqual((iso8601.parse_timezone(None).tzname(False), iso8601.parse_timezone(None).tzname(True)), time.tzname)
|
||||
self.assertEqual(iso8601.parse_timezone('Z').tzname(True), "UTC")
|
||||
self.assertEqual(iso8601.parse_timezone('Z').dst(True), datetime.timedelta(0))
|
||||
datetime.datetime(2007, 1, 25, 16, 0))
|
||||
|
|
|
|||
|
|
@ -278,8 +278,6 @@ class Transmitter(TransmitterBase):
|
|||
"datepattern", "Epoch", (None, "Epoch"), jail=self.jailName)
|
||||
self.setGetTest(
|
||||
"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
|
||||
self.setGetTest(
|
||||
"datepattern", "ISO8601", (None, "ISO8601"), jail=self.jailName)
|
||||
self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
|
||||
|
||||
def testJailUseDNS(self):
|
||||
|
|
|
|||
Loading…
Reference in a new issue