diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 25e15213..d6ad4e34 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -65,136 +65,7 @@ logSys = getLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] -class SafeConfigParserWithIncludes(object): - - SECTION_NAME = "INCLUDES" - CFG_CACHE = {} - CFG_INC_CACHE = {} - CFG_EMPY_CFG = None - - def __init__(self): - self.__cr = None - - def __check_read(self, attr): - if self.__cr is None: - # raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr) - if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None: - SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes() - self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG - - def __getattr__(self,attr): - # check we access local implementation - try: - orig_attr = self.__getattribute__(attr) - except AttributeError: - self.__check_read(attr) - orig_attr = self.__cr.__getattribute__(attr) - return orig_attr - - @staticmethod - def _get_resource_fingerprint(resource): - mt = [] - dirnames = [] - for filename in resource: - if os.path.exists(filename): - s = os.stat(filename) - mt.append(s.st_mtime) - mt.append(s.st_mode) - mt.append(s.st_size) - dirname = os.path.dirname(filename) - if dirname not in dirnames: - dirnames.append(dirname) - for dirname in dirnames: - if os.path.exists(dirname): - s = os.stat(dirname) - mt.append(s.st_mtime) - mt.append(s.st_mode) - mt.append(s.st_size) - return mt - - def read(self, resource, get_includes=True, log_info=None): - SCPWI = SafeConfigParserWithIncludes - # check includes : - fileNamesFull = [] - if not isinstance(resource, list): - resource = [ resource ] - if get_includes: - for filename in resource: - fileNamesFull += SCPWI.getIncludes(filename) - else: - fileNamesFull = resource - # check cache - hashv = '\x01'.join(fileNamesFull) - cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) - curmt = SCPWI._get_resource_fingerprint(fileNamesFull) - if cr is not None and mtime == curmt: - self.__cr = cr - logSys.debug("Cached config files: %s", resource) - #logSys.debug("Cached config files: %s", fileNamesFull) - return ret - # not yet in cache - create/read and add to cache: - if log_info is not None: - logSys.info(*log_info) - cr = _SafeConfigParserWithIncludes() - ret = cr.read(fileNamesFull) - SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt) - self.__cr = cr - return ret - - def getOptions(self, *args, **kwargs): - self.__check_read('getOptions') - return self.__cr.getOptions(*args, **kwargs) - - @staticmethod - def getIncludes(resource, seen = []): - """ - Given 1 config resource returns list of included files - (recursively) with the original one as well - Simple loops are taken care about - """ - - # Use a short class name ;) - SCPWI = SafeConfigParserWithIncludes - - resources = seen + [resource] - # check cache - hashv = '///'.join(resources) - cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0)) - curmt = SCPWI._get_resource_fingerprint(resources) - if cinc is not None and mtime == curmt: - return cinc - - parser = SCPWI() - try: - # read without includes - parser.read(resource, get_includes=False) - except UnicodeDecodeError, e: - logSys.error("Error decoding config file '%s': %s" % (resource, e)) - return [] - - resourceDir = os.path.dirname(resource) - - newFiles = [ ('before', []), ('after', []) ] - if SCPWI.SECTION_NAME in parser.sections(): - for option_name, option_list in newFiles: - if option_name in parser.options(SCPWI.SECTION_NAME): - newResources = parser.get(SCPWI.SECTION_NAME, option_name) - for newResource in newResources.split('\n'): - if os.path.isabs(newResource): - r = newResource - else: - r = os.path.join(resourceDir, newResource) - if r in seen: - continue - option_list += SCPWI.getIncludes(r, resources) - # combine lists - cinc = newFiles[0][1] + [resource] + newFiles[1][1] - # cache and return : - SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt) - return cinc - #print "Includes list for " + resource + " is " + `resources` - -class _SafeConfigParserWithIncludes(SafeConfigParser, object): +class SafeConfigParserWithIncludes(SafeConfigParser): """ Class adds functionality to SafeConfigParser to handle included other configuration files (or may be urls, whatever in the future) @@ -223,14 +94,99 @@ after = 1.conf """ + SECTION_NAME = "INCLUDES" + if sys.version_info >= (3,2): # overload constructor only for fancy new Python3's - def __init__(self, *args, **kwargs): + def __init__(self, share_config=None, *args, **kwargs): kwargs = kwargs.copy() kwargs['interpolation'] = BasicInterpolationWithName() kwargs['inline_comment_prefixes'] = ";" - super(_SafeConfigParserWithIncludes, self).__init__( + super(SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) + self._cfg_share = share_config + + else: + def __init__(self, share_config=None, *args, **kwargs): + SafeConfigParser.__init__(self, *args, **kwargs) + self._cfg_share = share_config + + @property + def share_config(self): + return self._cfg_share + + def _getSharedSCPWI(self, filename): + SCPWI = SafeConfigParserWithIncludes + # read single one, add to return list, use sharing if possible: + if self._cfg_share: + # cache/share each file as include (ex: filter.d/common could be included in each filter config): + hashv = 'inc:'+(filename if not isinstance(filename, list) else '\x01'.join(filename)) + cfg, i = self._cfg_share.get(hashv, (None, None)) + if cfg is None: + cfg = SCPWI(share_config=self._cfg_share) + i = cfg.read(filename, get_includes=False) + self._cfg_share[hashv] = (cfg, i) + else: + logSys.debug(" Shared file: %s", filename) + else: + # don't have sharing: + cfg = SCPWI() + i = cfg.read(filename, get_includes=False) + return (cfg, i) + + def _getIncludes(self, filenames, seen=[]): + if not isinstance(filenames, list): + filenames = [ filenames ] + # retrieve or cache include paths: + if self._cfg_share: + # cache/share include list: + hashv = 'inc-path:'+('\x01'.join(filenames)) + fileNamesFull = self._cfg_share.get(hashv) + if fileNamesFull is None: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + self._cfg_share[hashv] = fileNamesFull + return fileNamesFull + # don't have sharing: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + return fileNamesFull + + def __getIncludesUncached(self, resource, seen=[]): + """ + Given 1 config resource returns list of included files + (recursively) with the original one as well + Simple loops are taken care about + """ + SCPWI = SafeConfigParserWithIncludes + try: + parser, i = self._getSharedSCPWI(resource) + if not i: + return [] + except UnicodeDecodeError, e: + logSys.error("Error decoding config file '%s': %s" % (resource, e)) + return [] + + resourceDir = os.path.dirname(resource) + + newFiles = [ ('before', []), ('after', []) ] + if SCPWI.SECTION_NAME in parser.sections(): + for option_name, option_list in newFiles: + if option_name in parser.options(SCPWI.SECTION_NAME): + newResources = parser.get(SCPWI.SECTION_NAME, option_name) + for newResource in newResources.split('\n'): + if os.path.isabs(newResource): + r = newResource + else: + r = os.path.join(resourceDir, newResource) + if r in seen: + continue + s = seen + [resource] + option_list += self._getIncludes(r, s) + # combine lists + return newFiles[0][1] + [resource] + newFiles[1][1] def get_defaults(self): return self._defaults @@ -238,38 +194,52 @@ after = 1.conf def get_sections(self): return self._sections - def read(self, filenames): + def read(self, filenames, get_includes=True, log_info=None): if not isinstance(filenames, list): filenames = [ filenames ] - if len(filenames) > 1: - # read multiple configs: - ret = [] - alld = self.get_defaults() - alls = self.get_sections() - for filename in filenames: - # read single one, add to return list: - cfg = SafeConfigParserWithIncludes() - i = cfg.read(filename, get_includes=False) - if i: - ret += i - # merge defaults and all sections to self: - alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().iteritems(): - if isinstance(s, dict): - s2 = alls.get(n) - if isinstance(s2, dict): - s2.update(s) - else: - alls[n] = s.copy() - else: - alls[n] = s - - return ret - - # read one config : - logSys.debug("Reading file: %s", filenames[0]) - if sys.version_info >= (3,2): # pragma: no cover - return SafeConfigParser.read(self, filenames, encoding='utf-8') + # retrieve (and cache) includes: + fileNamesFull = [] + if get_includes: + fileNamesFull += self._getIncludes(filenames) else: - return SafeConfigParser.read(self, filenames) + fileNamesFull = filenames + + if self._cfg_share is not None: + logSys.debug(" Sharing files: %s", fileNamesFull) + + if len(fileNamesFull) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in fileNamesFull: + # read single one, add to return list, use sharing if possible: + cfg, i = self._getSharedSCPWI(filename) + if i: + ret += i + # merge defaults and all sections to self: + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): + if isinstance(s, dict): + s2 = alls.get(n) + if isinstance(s2, dict): + s2.update(s) + else: + alls[n] = s.copy() + else: + alls[n] = s + + return ret + + # read one config : + logSys.debug(" Reading file: %s", fileNamesFull[0]) + else: + # don't have sharing - read one or multiple at once: + logSys.debug(" Reading files: %s", fileNamesFull) + + # read file(s) : + if sys.version_info >= (3,2): # pragma: no cover + return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') + else: + return SafeConfigParser.read(self, fileNamesFull) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 096f8e78..657800fa 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -46,26 +46,38 @@ class ConfigReader(): self._cfg = None if use_config is not None: self._cfg = use_config - else: - # share config if possible: - if share_config is not None: - self._cfg_share = share_config - self._cfg_share_kwargs = kwargs - else: - self._cfg = ConfigReaderUnshared(**kwargs) + # share config if possible: + if share_config is not None: + self._cfg_share = share_config + self._cfg_share_kwargs = kwargs + self._cfg_share_basedir = None + elif self._cfg is None: + self._cfg = ConfigReaderUnshared(**kwargs) def setBaseDir(self, basedir): - self._cfg.setBaseDir(basedir) + if self._cfg: + self._cfg.setBaseDir(basedir) + else: + self._cfg_share_basedir = basedir def getBaseDir(self): - return self._cfg.getBaseDir() + if self._cfg: + return self._cfg.getBaseDir() + else: + return self._cfg_share_basedir + + @property + def share_config(self): + return self._cfg_share def read(self, name, once=True): # shared ? if not self._cfg and self._cfg_share is not None: self._cfg = self._cfg_share.get(name) if not self._cfg: - self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) + self._cfg = ConfigReaderUnshared(share_config=self._cfg_share, **self._cfg_share_kwargs) + if self._cfg_share_basedir is not None: + self._cfg.setBaseDir(self._cfg_share_basedir) self._cfg_share[name] = self._cfg # performance feature - read once if using shared config reader: rc = self._cfg.read_cfg_files @@ -73,6 +85,8 @@ class ConfigReader(): return rc.get(name) # read: + if self._cfg_share is not None: + logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir()) ret = self._cfg.read(name) # save already read: @@ -105,8 +119,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): DEFAULT_BASEDIR = '/etc/fail2ban' - def __init__(self, basedir=None): - SafeConfigParserWithIncludes.__init__(self) + def __init__(self, basedir=None, *args, **kwargs): + SafeConfigParserWithIncludes.__init__(self, *args, **kwargs) self.read_cfg_files = dict() self.setBaseDir(basedir) @@ -123,7 +137,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.debug("Reading configs for %s under %s " , filename, self._basedir) + logSys.info("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -140,8 +154,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if len(config_files): # at least one config exists and accessible logSys.debug("Reading config files: %s", ', '.join(config_files)) - config_files_read = SafeConfigParserWithIncludes.read(self, config_files, - log_info=("Cache configs for %s under %s " , filename, self._basedir)) + config_files_read = SafeConfigParserWithIncludes.read(self, config_files) missed = [ cf for cf in config_files if cf not in config_files_read ] if missed: logSys.error("Could not read config files: %s", ', '.join(missed)) diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index a29fe94d..4c0c0428 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,11 +33,11 @@ logSys = getLogger(__name__) class Configurator: - def __init__(self, force_enable=False): + def __init__(self, force_enable=False, share_config=None): self.__settings = dict() self.__streams = dict() - self.__fail2ban = Fail2banReader() - self.__jails = JailsReader(force_enable=force_enable) + self.__fail2ban = Fail2banReader(share_config=share_config) + self.__jails = JailsReader(force_enable=force_enable, share_config=share_config) def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index e4394e59..99891c0f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -41,13 +41,12 @@ class JailReader(ConfigReader): optionExtractRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') - def __init__(self, name, force_enable=False, cfg_share=None, **kwargs): + def __init__(self, name, force_enable=False, **kwargs): # use shared config if possible: ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None self.__force_enable = force_enable - self.__cfg_share = cfg_share self.__actions = list() self.__opts = None @@ -113,7 +112,7 @@ class JailReader(ConfigReader): filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( - filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir()) + filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -143,7 +142,7 @@ class JailReader(ConfigReader): else: action = ActionReader( actName, self.__name, actOpt, - share_config=self.__cfg_share, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 3bb713f6..7043aa67 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -43,7 +43,6 @@ class JailsReader(ConfigReader): """ # use shared config if possible: ConfigReader.__init__(self, **kwargs) - self.__cfg_share = dict() self.__jails = list() self.__force_enable = force_enable @@ -73,7 +72,7 @@ class JailsReader(ConfigReader): # use the cfg_share for filter/action caching and the same config for all # jails (use_config=...), therefore don't read it here: jail = JailReader(sec, force_enable=self.__force_enable, - cfg_share=self.__cfg_share, use_config=self._cfg) + share_config=self.share_config, use_config=self._cfg) ret = jail.getOptions() if ret: if jail.isEnabled(): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0aaa8890..dadbef0f 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -39,8 +39,6 @@ STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') -LAST_WRITE_TIME = 0 - class ConfigReaderTest(unittest.TestCase): def setUp(self): @@ -59,8 +57,7 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - fname = "%s/%s" % (self.d, fname) - f = open(fname, "w") + f = open("%s/%s" % (self.d, fname), "w") if value is not None: f.write(""" [section] @@ -69,14 +66,6 @@ option = %s if content is not None: f.write(content) f.close() - # set modification time to another second to revalidate cache (if milliseconds not supported) : - global LAST_WRITE_TIME - mtime = os.path.getmtime(fname) - if LAST_WRITE_TIME == mtime: - mtime += 1 - os.utime(fname, (mtime, mtime)) - LAST_WRITE_TIME = mtime - def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) @@ -102,6 +91,7 @@ option = %s # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) pass + def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet self._write("c.conf", "1") @@ -347,9 +337,9 @@ class FilterReaderTest(unittest.TestCase): class JailsReaderTestCache(LogCaptureTestCase): - def _readWholeConf(self, basedir, force_enable=False): + def _readWholeConf(self, basedir, force_enable=False, share_config=None): # read whole configuration like a file2ban-client ... - configurator = Configurator(force_enable=force_enable) + configurator = Configurator(force_enable=force_enable, share_config=share_config) configurator.setBaseDir(basedir) configurator.readEarly() configurator.getEarlyOptions() @@ -360,7 +350,7 @@ class JailsReaderTestCache(LogCaptureTestCase): def _getLoggedReadCount(self, filematch): cnt = 0 for s in self.getLog().rsplit('\n'): - if re.match(r"^Reading files?: .*/"+filematch, s): + if re.match(r"^\s*Reading files?: .*/"+filematch, s): cnt += 1 return cnt @@ -372,8 +362,11 @@ class JailsReaderTestCache(LogCaptureTestCase): shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local') shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') + # common sharing handle for this test: + share_cfg = dict() + # read whole configuration like a file2ban-client ... - self._readWholeConf(basedir) + self._readWholeConf(basedir, share_config=share_cfg) # how many times jail.local was read: cnt = self._getLoggedReadCount('jail.local') # if cnt > 1: @@ -382,7 +375,7 @@ class JailsReaderTestCache(LogCaptureTestCase): # read whole configuration like a file2ban-client, again ... # but this time force enable all jails, to check filter and action cached also: - self._readWholeConf(basedir, force_enable=True) + self._readWholeConf(basedir, force_enable=True, share_config=share_cfg) cnt = self._getLoggedReadCount(r'jail\.local') # still one (no more reads): self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt)