diff --git a/ChangeLog b/ChangeLog index 433026f3..76719f16 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released * New Actions: - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367) - Enhancements: + * Extreme speedup of all sqlite database operations (gh-1436), + by using of following sqlite options: + - (synchronous = OFF) write data through OS without syncing + - (journal_mode = MEMORY) use memory for the transaction logging + - (temp_store = MEMORY) temporary tables and indices are kept in memory * journald journalmatch for pure-ftpd (gh-1362) * Add additional regex filter for dovecot ldap authentication failures (gh-1370) * added additional regex filters for exim (gh-1371) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 7de87554..560fbfe5 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -181,8 +181,23 @@ class Fail2BanDb(object): filename, e.args[0]) raise + # differentiate pypy: switch journal mode later (save it during the upgrade), + # to prevent errors like "database table is locked": + try: + import __pypy__ + pypy = True + except ImportError: + pypy = False + cur = self._db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") + cur.execute("PRAGMA foreign_keys = ON") + # speedup: write data through OS without syncing (no wait): + cur.execute("PRAGMA synchronous = OFF") + # speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible): + if not pypy: + cur.execute("PRAGMA journal_mode = MEMORY") + # speedup: temporary tables and indices are kept in memory: + cur.execute("PRAGMA temp_store = MEMORY") try: cur.execute("SELECT version FROM fail2banDb LIMIT 1") @@ -202,6 +217,9 @@ class Fail2BanDb(object): Fail2BanDb.__version__, version, newversion) raise RuntimeError('Failed to fully update') finally: + # pypy: set journal mode after possible upgrade db: + if pypy: + cur.execute("PRAGMA journal_mode = MEMORY") cur.close() @property @@ -244,13 +262,14 @@ class Fail2BanDb(object): A timestamped backup is also created prior to attempting the update. """ - self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) - shutil.copyfile(self.filename, self._dbBackupFilename) - logSys.info("Database backup created: %s", self._dbBackupFilename) if version > Fail2BanDb.__version__: raise NotImplementedError( "Attempt to travel to future version of database ...how did you get here??") + self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) + shutil.copyfile(self.filename, self._dbBackupFilename) + logSys.info("Database backup created: %s", self._dbBackupFilename) + if version < 2: cur.executescript("BEGIN TRANSACTION;" "CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"