From af078532ac3822a4b42dc008417aa9644d75606a Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 00:42:01 -0500 Subject: [PATCH 01/10] New jail: apache-fakegooglebot Detects fake googlebot user agents in apache access log --- ChangeLog | 3 +++ config/filter.d/apache-fakegooglebot.conf | 14 +++++++++++++ .../ignorecommands/apache-fakegooglebot.sh | 21 +++++++++++++++++++ config/jail.conf | 8 +++++++ config/paths-common.conf | 3 +++ fail2ban/client/filterreader.py | 2 +- .../tests/files/logs/apache-fakegooglebot | 5 +++++ fail2ban/tests/samplestestcase.py | 3 ++- setup.py | 5 ++++- 9 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 config/filter.d/apache-fakegooglebot.conf create mode 100755 config/filter.d/ignorecommands/apache-fakegooglebot.sh create mode 100644 fail2ban/tests/files/logs/apache-fakegooglebot diff --git a/ChangeLog b/ChangeLog index f1963b1e..a3370146 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,7 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - New Features: - New filter: - postfix-rbl Thanks Lee Clemens + - apache-fakegooglebot.conf Thanks Lee Clemens - New recursive embedded substitution feature added: - `<HOST>` becomes `` for PREF=`IPV4`; - `<HOST>` becomes `1.2.3.4` for PREF=`IPV4` and IPV4HOST=`1.2.3.4`; @@ -49,6 +50,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - empty or "basic" works as-is - "cymru" additionally prints (ASN, Country RIR) per banned IP - Requires dnspython or dnspython3 + - New ignorecommand: + - apache-fakegooglebot.sh Thanks Lee Clemens - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf new file mode 100644 index 00000000..b8a73504 --- /dev/null +++ b/config/filter.d/apache-fakegooglebot.conf @@ -0,0 +1,14 @@ +# Fail2Ban filter for fake Googlebot User Agents + +[Definition] + +failregex = ^ .*Googlebot.*$ + +ignoreregex = + + +# DEV Notes: +# +# Author: Lee Clemens +# Thanks: Johannes B. Ullrich, Ph.D. +# Reference: https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/ diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot.sh b/config/filter.d/ignorecommands/apache-fakegooglebot.sh new file mode 100755 index 00000000..d8b2d328 --- /dev/null +++ b/config/filter.d/ignorecommands/apache-fakegooglebot.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Based on: https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/ + +if [ "$#" -ne 1 ]; then + echo "Unexpected number of arguments: $#" + exit 1 +else + b="$1" + h=$(host ${b}) + if echo ${h} | grep -e ' crawl-.*\.googlebot\.com\.$'; then + h=$(echo ${h} | cut -f5 -d' ') + n=$(host ${h} | cut -f4 -d' ') + if [ "${n}" = "${b}" ] ; then + exit 0 + else + exit 1 + fi + else + exit 1 + fi +fi diff --git a/config/jail.conf b/config/jail.conf index 334e3298..621c7230 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -277,6 +277,14 @@ logpath = %(apache_error_log)s maxretry = 2 +[apache-fakegooglebot] + +port = http,https +logpath = %(apache_access_log)s +maxretry = 1 +ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot.sh + + [apache-modsecurity] port = http,https diff --git a/config/paths-common.conf b/config/paths-common.conf index c634e73d..837d339a 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -61,3 +61,6 @@ dovecot_log = %(syslog_mail_warn)s solidpop3d_log = %(syslog_local0)s mysql_log = %(syslog_daemon)s + +# Directory with ignorecommand scripts +ignorecommands_dir = /etc/fail2ban/filter.d/ignorecommands diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 669a5896..51d18c0e 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -71,7 +71,7 @@ class FilterReader(DefinitionInitConfigReader): for regex in value.split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self._jailName, "addignoreregex", regex]) + stream.append(["set", self._jailName, "addignoreregex", regex]) if self._initOpts: if 'maxlines' in self._initOpts: # We warn when multiline regex is used without maxlines > 1 diff --git a/fail2ban/tests/files/logs/apache-fakegooglebot b/fail2ban/tests/files/logs/apache-fakegooglebot new file mode 100644 index 00000000..76bb0dcb --- /dev/null +++ b/fail2ban/tests/files/logs/apache-fakegooglebot @@ -0,0 +1,5 @@ +# Apache 2.2 +# failJSON: { "time": "2015-01-31T14:29:44", "match": true, "host": "66.249.66.1" } +66.249.66.1 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + 293 1149 546 +# failJSON: { "time": "2015-01-31T14:29:44", "match": false, "host": "93.184.216.34" } +93.184.216.34 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "NOT A __GOOGLE_BOT__" + 293 1149 546 diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 2c18a504..bd2bd2e7 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -141,7 +141,8 @@ def testSampleRegexsFactory(name): return testFilter -for filter_ in filter(lambda x: not x.endswith('common.conf'), os.listdir(os.path.join(CONFIG_DIR, "filter.d"))): +for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'), + os.listdir(os.path.join(CONFIG_DIR, "filter.d"))): filterName = filter_.rpartition(".")[0] if not filterName.startswith('.'): setattr( diff --git a/setup.py b/setup.py index f0b4b331..a6970eff 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ setup( 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', + 'bin/fail2ban-testcases' ], packages = [ 'fail2ban', @@ -124,6 +124,9 @@ setup( ('/etc/fail2ban/filter.d', glob("config/filter.d/*.conf") ), + ('/etc/fail2ban/filter.d/ignorecommands', + glob("config/filter.d/ignorecommands/*") + ), ('/etc/fail2ban/action.d', glob("config/action.d/*.conf") + glob("config/action.d/*.py") From 38415d2cf185628ebe40814283461bc07e76ebe1 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 11:29:27 -0500 Subject: [PATCH 02/10] Add trailing comma back to list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6970eff..5f497a3e 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ setup( 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases' + 'bin/fail2ban-testcases', ], packages = [ 'fail2ban', From 00961d52819bd4b8e13e5fbbb36a2930ab6ebd88 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 11:36:21 -0500 Subject: [PATCH 03/10] Remove ignorecommand addition from ChangeLog --- ChangeLog | 2 -- 1 file changed, 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index a3370146..65c0ecb5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -50,8 +50,6 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - empty or "basic" works as-is - "cymru" additionally prints (ASN, Country RIR) per banned IP - Requires dnspython or dnspython3 - - New ignorecommand: - - apache-fakegooglebot.sh Thanks Lee Clemens - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 From 854915920f1b758ad646ff853955b6aebba4b641 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 11:38:23 -0500 Subject: [PATCH 04/10] Remove implementation specific suffix --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 621c7230..73ddd9c6 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -282,7 +282,7 @@ maxretry = 2 port = http,https logpath = %(apache_access_log)s maxretry = 1 -ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot.sh +ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot [apache-modsecurity] From 7e94ba6f0c12cca5c118c2dea092ea708ec5f72e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 11:43:05 -0500 Subject: [PATCH 05/10] Remove implementation specific suffix --- .../{apache-fakegooglebot.sh => apache-fakegooglebot} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config/filter.d/ignorecommands/{apache-fakegooglebot.sh => apache-fakegooglebot} (100%) diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot.sh b/config/filter.d/ignorecommands/apache-fakegooglebot similarity index 100% rename from config/filter.d/ignorecommands/apache-fakegooglebot.sh rename to config/filter.d/ignorecommands/apache-fakegooglebot From 0463518d28a4146bce7c3bb516951c77bf0c392a Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 13:14:53 -0500 Subject: [PATCH 06/10] Add testcase for DNSUtils.ipToName --- fail2ban/tests/filtertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 47e88067..2274b9e5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -996,6 +996,8 @@ class DNSUtilsTests(unittest.TestCase): self.assertEqual(res, ['93.184.216.34']) res = DNSUtils.textToIp('www.example.com', 'yes') self.assertEqual(res, ['93.184.216.34']) + res = DNSUtils.ipToName('66.249.66.1') + self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') def testTextToIp(self): # Test hostnames From 4cfbb840ad554d527112a5589e21c0377b00d9be Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 13:40:56 -0500 Subject: [PATCH 07/10] move test to more sensible function name --- fail2ban/tests/filtertestcase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2274b9e5..1368db32 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -996,8 +996,6 @@ class DNSUtilsTests(unittest.TestCase): self.assertEqual(res, ['93.184.216.34']) res = DNSUtils.textToIp('www.example.com', 'yes') self.assertEqual(res, ['93.184.216.34']) - res = DNSUtils.ipToName('66.249.66.1') - self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') def testTextToIp(self): # Test hostnames @@ -1013,6 +1011,10 @@ class DNSUtilsTests(unittest.TestCase): else: self.assertEqual(res, []) + def testIpToName(self): + res = DNSUtils.ipToName('66.249.66.1') + self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') + class JailTests(unittest.TestCase): def testSetBackend_gh83(self): From fef031b3cd41c99a4843d5d5b52217b7694eff72 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 14:01:48 -0500 Subject: [PATCH 08/10] test ipToName Exception --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 1368db32..1146cef8 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1014,6 +1014,7 @@ class DNSUtilsTests(unittest.TestCase): def testIpToName(self): res = DNSUtils.ipToName('66.249.66.1') self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') + self.assertRaises(Exception, DNSUtils.ipToName, ['10.0.0.0']) class JailTests(unittest.TestCase): From eb17b68641aa7b3187205cd7818148d4ee1562cd Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 14:03:35 -0500 Subject: [PATCH 09/10] fix test for invalid IP --- fail2ban/tests/filtertestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 1146cef8..32c89dbf 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1014,7 +1014,8 @@ class DNSUtilsTests(unittest.TestCase): def testIpToName(self): res = DNSUtils.ipToName('66.249.66.1') self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') - self.assertRaises(Exception, DNSUtils.ipToName, ['10.0.0.0']) + res = DNSUtils.ipToName('10.0.0.0') + self.assertEqual(res, None) class JailTests(unittest.TestCase): From b725db04d861b506d4059b9613a2d139c8f8b5d9 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Mon, 2 Feb 2015 15:03:44 -0500 Subject: [PATCH 10/10] Add ignorecommands to MANIFEST*'s --- MANIFEST | 2 ++ MANIFEST.in | 1 + 2 files changed, 3 insertions(+) diff --git a/MANIFEST b/MANIFEST index c6de80fd..d6ab1156 100644 --- a/MANIFEST +++ b/MANIFEST @@ -266,6 +266,8 @@ config/filter.d/groupoffice.conf config/filter.d/gssftpd.conf config/filter.d/guacamole.conf config/filter.d/horde.conf +config/filter.d/ignorecommands +config/filter.d/ignorecommands/apache-fakegooglebot config/filter.d/kerio.conf config/filter.d/lighttpd-auth.conf config/filter.d/monit.conf diff --git a/MANIFEST.in b/MANIFEST.in index 9473d3a6..22696a20 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include ChangeLog COPYING DEVELOP FILTERS README.* THANKS TODO CONTRIBUTING* Vag graft doc graft files recursive-include config *.conf *.py +recursive-include config/filter.d/ignorecommands *