From db9e7240386b7fdee5d796138f7118d01e3168e0 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 20 May 2016 12:03:35 +0200 Subject: [PATCH 1/3] extremely speedup of all database operations: - (synchronous = OFF) write data through OS without syncing - (journal_mode = MEMORY) use memory for the transaction logging --- fail2ban/server/database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 7de87554..c0e9bf46 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -182,8 +182,11 @@ class Fail2BanDb(object): raise cur = self._db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") - + cur.execute("PRAGMA foreign_keys = ON") + # speedup: write data through OS without syncing (no wait): + cur.execute("PRAGMA synchronous = OFF") + # speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible): + cur.execute("PRAGMA journal_mode = MEMORY") try: cur.execute("SELECT version FROM fail2banDb LIMIT 1") except sqlite3.OperationalError: From baafac36a4c578769d53240147d2777b6414ad1c Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 20 May 2016 12:42:18 +0200 Subject: [PATCH 2/3] ChangeLog entry --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 433026f3..76719f16 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released * New Actions: - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367) - Enhancements: + * Extreme speedup of all sqlite database operations (gh-1436), + by using of following sqlite options: + - (synchronous = OFF) write data through OS without syncing + - (journal_mode = MEMORY) use memory for the transaction logging + - (temp_store = MEMORY) temporary tables and indices are kept in memory * journald journalmatch for pure-ftpd (gh-1362) * Add additional regex filter for dovecot ldap authentication failures (gh-1370) * added additional regex filters for exim (gh-1371) From 1718c8dbe92030617042db2c7a01e996cf16cab7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 20 May 2016 13:26:51 +0200 Subject: [PATCH 3/3] pypy: switch journal mode after upgrade (save it during the upgrade), to prevent errors like "database table is locked" --- fail2ban/server/database.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index c0e9bf46..560fbfe5 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -181,12 +181,24 @@ class Fail2BanDb(object): filename, e.args[0]) raise + # differentiate pypy: switch journal mode later (save it during the upgrade), + # to prevent errors like "database table is locked": + try: + import __pypy__ + pypy = True + except ImportError: + pypy = False + cur = self._db.cursor() cur.execute("PRAGMA foreign_keys = ON") # speedup: write data through OS without syncing (no wait): cur.execute("PRAGMA synchronous = OFF") # speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible): - cur.execute("PRAGMA journal_mode = MEMORY") + if not pypy: + cur.execute("PRAGMA journal_mode = MEMORY") + # speedup: temporary tables and indices are kept in memory: + cur.execute("PRAGMA temp_store = MEMORY") + try: cur.execute("SELECT version FROM fail2banDb LIMIT 1") except sqlite3.OperationalError: @@ -205,6 +217,9 @@ class Fail2BanDb(object): Fail2BanDb.__version__, version, newversion) raise RuntimeError('Failed to fully update') finally: + # pypy: set journal mode after possible upgrade db: + if pypy: + cur.execute("PRAGMA journal_mode = MEMORY") cur.close() @property @@ -247,13 +262,14 @@ class Fail2BanDb(object): A timestamped backup is also created prior to attempting the update. """ - self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) - shutil.copyfile(self.filename, self._dbBackupFilename) - logSys.info("Database backup created: %s", self._dbBackupFilename) if version > Fail2BanDb.__version__: raise NotImplementedError( "Attempt to travel to future version of database ...how did you get here??") + self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) + shutil.copyfile(self.filename, self._dbBackupFilename) + logSys.info("Database backup created: %s", self._dbBackupFilename) + if version < 2: cur.executescript("BEGIN TRANSACTION;" "CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"