diff --git a/ChangeLog b/ChangeLog index e43d0b2b..4cf1d106 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released system authentication issues * fail2ban-regex reads filter file(s) completely, incl. '.local' file etc. (gh-954) * firewallcmd-* actions: split output into separate lines for grepping (gh-908) + * Guard unicode encode/decode issues while storing records in the database. + Fixes "binding parameter error (unsupported type)" (gh-973), thanks to kot for reporting - New Features: - New filters: diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index bbf7adad..4e38b6ed 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -37,17 +37,49 @@ from ..helpers import getLogger logSys = getLogger(__name__) if sys.version_info >= (3,): - sqlite3.register_adapter( - dict, - lambda x: json.dumps(x, ensure_ascii=False).encode( - locale.getpreferredencoding(), 'replace')) - sqlite3.register_converter( - "JSON", - lambda x: json.loads(x.decode( - locale.getpreferredencoding(), 'replace'))) + def _json_dumps_safe(x): + try: + x = json.dumps(x, ensure_ascii=False).encode( + locale.getpreferredencoding(), 'replace') + except Exception, e: # pragma: no cover + logSys.error('json dumps failed: %s', e) + x = '{}' + return x + def _json_loads_safe(x): + try: + x = json.loads(x.decode(locale.getpreferredencoding(), 'replace')) + except Exception, e: # pragma: no cover + logSys.error('json loads failed: %s', e) + x = {} + return x else: - sqlite3.register_adapter(dict, json.dumps) - sqlite3.register_converter("JSON", json.loads) + def _normalize(x): + if isinstance(x, dict): + return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems()) + elif isinstance(x, list): + return [_normalize(element) for element in x] + elif isinstance(x, unicode): + return x.encode(locale.getpreferredencoding()) + else: + return x + def _json_dumps_safe(x): + try: + x = json.dumps(_normalize(x), ensure_ascii=False).decode( + locale.getpreferredencoding(), 'replace') + except Exception, e: # pragma: no cover + logSys.error('json dumps failed: %s', e) + x = '{}' + return x + def _json_loads_safe(x): + try: + x = _normalize(json.loads(x.decode(locale.getpreferredencoding(), 'replace'))) + except Exception, e: # pragma: no cover + logSys.error('json loads failed: %s', e) + x = {} + return x + +sqlite3.register_adapter(dict, _json_dumps_safe) +sqlite3.register_converter("JSON", _json_loads_safe) def commitandrollback(f): @wraps(f) @@ -431,8 +463,8 @@ class Fail2BanDb(object): tickets = [] for ip, timeofban, data in self._getBans(**kwargs): #TODO: Implement data parts once arbitrary match keys completed - tickets.append(FailTicket(ip, timeofban, data['matches'])) - tickets[-1].setAttempt(data['failures']) + tickets.append(FailTicket(ip, timeofban, data.get('matches'))) + tickets[-1].setAttempt(data.get('failures', 1)) return tickets def getBansMerged(self, ip=None, jail=None, bantime=None): @@ -484,8 +516,8 @@ class Fail2BanDb(object): prev_banip = banip matches = [] failures = 0 - matches.extend(data['matches']) - failures += data['failures'] + matches.extend(data.get('matches', [])) + failures += data.get('failures', 1) prev_timeofban = timeofban ticket = FailTicket(banip, prev_timeofban, matches) ticket.setAttempt(failures) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 22866841..cf0c0e2b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -802,8 +802,8 @@ class FileContainer: " encoding) for this jail. Continuing" " to process line ignoring invalid characters: %r" % (self.getFileName(), self.getEncoding(), line)) - if sys.version_info >= (3,): # In python3, must be decoded - line = line.decode(self.getEncoding(), 'ignore') + # decode with replacing error chars: + line = line.decode(self.getEncoding(), 'replace') return line def close(self): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index aef7443f..9665e322 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -177,6 +177,37 @@ class DatabaseTest(LogCaptureTestCase): self.assertTrue( isinstance(self.db.getBans(jail=self.jail)[0], FailTicket)) + def testAddBanInvalidEncoded(self): + if Fail2BanDb is None: # pragma: no cover + return + self.testAddJail() + # invalid + valid, invalid + valid unicode, invalid + valid dual converted (like in filter:readline by fallback) ... + tickets = [ + FailTicket("127.0.0.1", 0, ['user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), + FailTicket("127.0.0.2", 0, ['user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), + FailTicket("127.0.0.3", 0, ['user "\xd1\xe2\xe5\xf2\xe0"', b'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"'.decode('utf-8', 'replace')]) + ] + self.db.addBan(self.jail, tickets[0]) + self.db.addBan(self.jail, tickets[1]) + self.db.addBan(self.jail, tickets[2]) + + readtickets = self.db.getBans(jail=self.jail) + self.assertEqual(len(readtickets), 3) + ## python 2 or 3 : + invstr = u'user "\ufffd\ufffd\ufffd\ufffd\ufffd"'.encode('utf-8', 'replace') + self.assertTrue( + readtickets[0] == FailTicket("127.0.0.1", 0, [invstr, 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']) + or readtickets[0] == tickets[0] + ) + self.assertTrue( + readtickets[1] == FailTicket("127.0.0.2", 0, [invstr, u'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"'.encode('utf-8', 'replace')]) + or readtickets[1] == tickets[1] + ) + self.assertTrue( + readtickets[2] == FailTicket("127.0.0.3", 0, [invstr, 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']) + or readtickets[2] == tickets[2] + ) + def testDelBan(self): self.testAddBan() ticket = self.db.getBans(jail=self.jail)[0]