From d9248a6cf87332534950748782d3ce06df3f5424 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 19 Jul 2012 17:26:09 -0400 Subject: [PATCH] BF+RF: pyinotify refreshes watcher upon CREATE, unified/simplified *(add|del)LogPath among *Filters * all of the *Filters had too much of common logic in their *LogPath methods, which is now handled by FileFilter and derived classes only add custom actions in corresponding _(add|del)LogPath methods pyinotify: * upon CREATE event: - unknown files should not be handled at all - "watcher" for the monitored files should be recreated. Lead to adding _(add|del)FileWatcher helper methods * callback now obtains full event to judge what to do --- server/filter.py | 22 ++++++++- server/filtergamin.py | 22 +++------ server/filterpoll.py | 22 +++------ server/filterpyinotify.py | 101 +++++++++++++++++++++++--------------- 4 files changed, 94 insertions(+), 73 deletions(-) diff --git a/server/filter.py b/server/filter.py index a9bc35f3..d69f02aa 100644 --- a/server/filter.py +++ b/server/filter.py @@ -395,8 +395,19 @@ class FileFilter(Filter): # @param path log file path def addLogPath(self, path, tail = False): - container = FileContainer(path, tail) - self.__logPath.append(container) + if self.containsLogPath(path): + logSys.error(path + " already exists") + else: + container = FileContainer(path, tail) + self.__logPath.append(container) + logSys.info("Added logfile = %s" % path) + self._addLogPath(path) # backend specific + + def _addLogPath(self, path): + # nothing to do by default + # to be overriden by backends + pass + ## # Delete a log path @@ -407,8 +418,15 @@ class FileFilter(Filter): for log in self.__logPath: if log.getFileName() == path: self.__logPath.remove(log) + logSys.info("Removed logfile = %s" % path) + self._delLogPath(path) return + def _delLogPath(self, path): + # nothing to do by default + # to be overriden by backends + pass + ## # Get the log file path # diff --git a/server/filtergamin.py b/server/filtergamin.py index c1a6be3c..84148b1a 100644 --- a/server/filtergamin.py +++ b/server/filtergamin.py @@ -72,27 +72,17 @@ class FilterGamin(FileFilter): # # @param path log file path - def addLogPath(self, path, tail = False): - if self.containsLogPath(path): - logSys.error(path + " already exists") - else: - self.monitor.watch_file(path, self.callback) - FileFilter.addLogPath(self, path, tail) - logSys.info("Added logfile = %s" % path) - + def _addLogPath(self, path): + self.monitor.watch_file(path, self.callback) + ## # Delete a log path # # @param path the log file to delete - + def delLogPath(self, path): - if not self.containsLogPath(path): - logSys.error(path + " is not monitored") - else: - self.monitor.stop_watch(path) - FileFilter.delLogPath(self, path) - logSys.info("Removed logfile = %s" % path) - + self.monitor.stop_watch(path) + ## # Main loop. # diff --git a/server/filterpoll.py b/server/filterpoll.py index 81edf813..0719545a 100644 --- a/server/filterpoll.py +++ b/server/filterpoll.py @@ -63,28 +63,18 @@ class FilterPoll(FileFilter): # # @param path log file path - def addLogPath(self, path, tail = False): - if self.containsLogPath(path): - logSys.error(path + " already exists") - else: - self.__lastModTime[path] = 0 - self.__file404Cnt[path] = 0 - FileFilter.addLogPath(self, path, tail) - logSys.info("Added logfile = %s" % path) + def _addLogPath(self, path): + self.__lastModTime[path] = 0 + self.__file404Cnt[path] = 0 ## # Delete a log path # # @param path the log file to delete - def delLogPath(self, path): - if not self.containsLogPath(path): - logSys.error(path + " is not monitored") - else: - del self.__lastModTime[path] - del self.__file404Cnt[path] - FileFilter.delLogPath(self, path) - logSys.info("Removed logfile = %s" % path) + def _delLogPath(self, path): + del self.__lastModTime[path] + del self.__file404Cnt[path] ## # Main loop. diff --git a/server/filterpyinotify.py b/server/filterpyinotify.py index b07dbba1..008f9303 100644 --- a/server/filterpyinotify.py +++ b/server/filterpyinotify.py @@ -57,7 +57,29 @@ class FilterPyinotify(FileFilter): logSys.debug("Created FilterPyinotify") - def callback(self, path): + def callback(self, event): + path = event.pathname + if event.mask == pyinotify.IN_CREATE: + # check if that is a file we care about + if not path in self.__watches: + logSys.debug("Ignoring creation of %s we do not monitor" % path) + return + else: + # we need to substitute the watcher with a new one, so first + # remove old one + self._delFileWatcher(path) + # place a new one + self._addFileWatcher(path) + + self._process_file(path) + + + def _process_file(self, path): + """Process a given file + + TODO -- RF: + this is a common logic and must be shared/provided by FileFilter + """ self.getFailures(path) try: while True: @@ -68,57 +90,58 @@ class FilterPyinotify(FileFilter): self.dateDetector.sortTemplate() self.__modified = False + + def _addFileWatcher(self, path): + wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY) + self.__watches.update(wd) + logSys.debug("Added file watcher for %s" % path) + # process the file since we did get even + self._process_file(path) + + + def _delFileWatcher(self, path): + wdInt = self.__watches[path] + wd = self.__monitor.rm_watch(wdInt) + if wd[wdInt]: + del self.__watches[path] + logSys.debug("Removed file watcher for %s" % path) + return True + else: + return False + ## # Add a log file path # # @param path log file path - def addLogPath(self, path, tail=False): - if self.containsLogPath(path): - logSys.error(path + " already exists") - else: - wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY) - self.__watches.update(wd) + def _addLogPath(self, path): + path_dir = dirname(path) + if not (path_dir in self.__watches): + # we need to watch also the directory for IN_CREATE + self.__watches.update( + self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE)) + logSys.debug("Added monitor for the parent directory %s" % path_dir) - FileFilter.addLogPath(self, path, tail) - logSys.info("Added logfile = %s" % path) + self._addFileWatcher(path) - path_dir = dirname(path) - if not (path_dir in self.__watches): - # we need to watch also the directory for IN_CREATE - self.__watches.update( - self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE)) - logSys.debug("Monitor also parent directory %s" % path_dir) - - # sniff the file - self.callback(path) ## # Delete a log path # # @param path the log file to delete - def delLogPath(self, path): - if not self.containsLogPath(path): - logSys.error(path + " is not monitored") - else: - wdInt = self.__watches[path] - wd = self.__monitor.rm_watch(wdInt) - if wd[wdInt]: - del self.__watches[path] - FileFilter.delLogPath(self, path) - logSys.info("Removed logfile = %s" % path) - else: - logSys.error("Failed to remove watch on path: %s", path) + def _delLogPath(self, path): + if not self._delFileWatcher(path): + logSys.error("Failed to remove watch on path: %s", path) - path_dir = dirname(path) - if not len([k for k in self.__watches - if k.startswith(path_dir + pathsep)]): - # Remove watches for the directory - # since there is no other monitored file under this directory - wdInt = self.__watches.pop(path_dir) - _ = self.__monitor.rm_watch(wdInt) - logSys.debug("Remove monitor for the parent directory %s" % path_dir) + path_dir = dirname(path) + if not len([k for k in self.__watches + if k.startswith(path_dir + pathsep)]): + # Remove watches for the directory + # since there is no other monitored file under this directory + wdInt = self.__watches.pop(path_dir) + _ = self.__monitor.rm_watch(wdInt) + logSys.debug("Removed monitor for the parent directory %s" % path_dir) ## @@ -169,4 +192,4 @@ class ProcessPyinotify(pyinotify.ProcessEvent): # just need default, since using mask on watch to limit events def process_default(self, event): logSys.debug("Callback for Event: %s" % event) - self.__FileFilter.callback(event.pathname) + self.__FileFilter.callback(event)