From ad3e0d97c47305ddd7bceae85dab6a3cab3404df Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 12 Jan 2020 23:21:29 +0100 Subject: [PATCH] New upstream version 0.11.1 --- .travis.yml | 63 +- ChangeLog | 192 +++++- MANIFEST | 15 +- README.md | 28 +- THANKS | 2 +- config/action.d/abuseipdb.conf | 10 +- config/action.d/badips.py | 55 +- config/action.d/firewallcmd-ipset.conf | 10 +- config/action.d/helpers-common.conf | 5 +- .../iptables-ipset-proto6-allports.conf | 12 +- config/action.d/iptables-ipset-proto6.conf | 12 +- config/action.d/mail-whois-common.conf | 2 +- config/action.d/nftables-allports.conf | 11 +- config/action.d/nftables-common.conf | 135 ---- config/action.d/nftables-multiport.conf | 11 +- config/action.d/nftables.conf | 203 ++++++ config/action.d/nginx-block-map.conf | 2 +- config/action.d/osx-afctl.conf | 4 +- config/action.d/sendmail-buffered.conf | 8 +- config/action.d/sendmail-common.conf | 8 +- config/action.d/sendmail-geoip-lines.conf | 4 +- .../sendmail-whois-ipjailmatches.conf | 5 +- config/action.d/sendmail-whois-ipmatches.conf | 5 +- config/action.d/sendmail-whois-lines.conf | 9 +- config/action.d/sendmail-whois-matches.conf | 5 +- config/action.d/sendmail-whois.conf | 5 +- config/action.d/sendmail.conf | 2 +- config/action.d/shorewall-ipset-proto6.conf | 14 +- config/action.d/smtp.py | 12 +- config/action.d/xarf-login-attack.conf | 28 +- config/fail2ban.conf | 21 +- config/filter.d/apache-auth.conf | 10 + config/filter.d/apache-common.conf | 4 +- config/filter.d/apache-modsecurity.conf | 2 +- config/filter.d/apache-noscript.conf | 2 +- config/filter.d/asterisk.conf | 27 +- config/filter.d/bitwarden.conf | 6 + config/filter.d/centreon.conf | 9 + config/filter.d/common.conf | 28 +- .../ignorecommands/apache-fakegooglebot | 2 +- config/filter.d/mysqld-auth.conf | 4 +- config/filter.d/named-refused.conf | 8 +- config/filter.d/postfix.conf | 21 +- config/filter.d/sendmail-auth.conf | 4 +- config/filter.d/sendmail-reject.conf | 7 +- config/filter.d/sshd.conf | 20 +- config/filter.d/traefik-auth.conf | 56 ++ config/filter.d/znc-adminlog.conf | 34 + config/jail.conf | 75 ++- config/paths-debian.conf | 2 - fail2ban/client/actionreader.py | 5 +- fail2ban/client/beautifier.py | 6 + fail2ban/client/configreader.py | 5 + fail2ban/client/fail2banclient.py | 3 +- fail2ban/client/fail2banreader.py | 14 +- fail2ban/client/fail2banregex.py | 145 ++-- fail2ban/client/filterreader.py | 27 +- fail2ban/client/jailreader.py | 109 +-- fail2ban/helpers.py | 38 +- fail2ban/protocol.py | 10 +- fail2ban/server/action.py | 436 ++++++++---- fail2ban/server/actions.py | 226 ++++++- fail2ban/server/banmanager.py | 51 +- fail2ban/server/database.py | 213 ++++-- fail2ban/server/datedetector.py | 34 +- fail2ban/server/datetemplate.py | 21 +- fail2ban/server/failmanager.py | 26 +- fail2ban/server/failregex.py | 32 +- fail2ban/server/filter.py | 54 +- fail2ban/server/filtergamin.py | 9 +- fail2ban/server/filterpoll.py | 15 +- fail2ban/server/filterpyinotify.py | 9 +- fail2ban/server/filtersystemd.py | 14 +- fail2ban/server/ipdns.py | 42 +- fail2ban/server/jail.py | 94 ++- fail2ban/server/jailthread.py | 15 +- fail2ban/server/mytime.py | 10 + fail2ban/server/observer.py | 529 +++++++++++++++ fail2ban/server/server.py | 79 ++- fail2ban/server/ticket.py | 87 ++- fail2ban/server/transmitter.py | 147 +++- fail2ban/server/utils.py | 50 +- fail2ban/tests/action_d/test_badips.py | 3 +- fail2ban/tests/actionstestcase.py | 375 ++++++++++- fail2ban/tests/actiontestcase.py | 114 ++-- fail2ban/tests/banmanagertestcase.py | 25 + fail2ban/tests/clientreadertestcase.py | 54 +- .../filter.d/zzz-sshd-obsolete-multiline.conf | 1 + fail2ban/tests/config/jail.conf | 13 +- fail2ban/tests/databasetestcase.py | 96 ++- fail2ban/tests/datedetectortestcase.py | 4 +- fail2ban/tests/dummyjail.py | 15 +- fail2ban/tests/fail2banclienttestcase.py | 340 +++++++++- fail2ban/tests/fail2banregextestcase.py | 303 +++++---- fail2ban/tests/failmanagertestcase.py | 32 +- .../files/action.d/action_modifyainfo.py | 5 + fail2ban/tests/files/database_v1.db | Bin 15360 -> 15360 bytes fail2ban/tests/files/database_v2.db | Bin 0 -> 16384 bytes fail2ban/tests/files/logs/apache-auth | 8 + fail2ban/tests/files/logs/apache-modsecurity | 3 + fail2ban/tests/files/logs/apache-noscript | 4 +- fail2ban/tests/files/logs/asterisk | 13 + fail2ban/tests/files/logs/bitwarden | 5 + fail2ban/tests/files/logs/centreon | 4 + fail2ban/tests/files/logs/monit | 3 + fail2ban/tests/files/logs/murmur | 5 + fail2ban/tests/files/logs/mysqld-auth | 5 + fail2ban/tests/files/logs/named-refused | 3 + fail2ban/tests/files/logs/postfix | 20 +- fail2ban/tests/files/logs/sendmail-auth | 3 + fail2ban/tests/files/logs/sendmail-reject | 8 + fail2ban/tests/files/logs/sshd | 17 + fail2ban/tests/files/logs/sshd-journal | 346 ++++++++++ fail2ban/tests/files/logs/traefik-auth | 6 + fail2ban/tests/files/logs/znc-adminlog | 15 + fail2ban/tests/filtertestcase.py | 217 ++++-- fail2ban/tests/observertestcase.py | 626 ++++++++++++++++++ fail2ban/tests/samplestestcase.py | 78 ++- fail2ban/tests/servertestcase.py | 384 +++++++++-- fail2ban/tests/sockettestcase.py | 1 + fail2ban/tests/utils.py | 39 +- fail2ban/version.py | 2 +- files/debian-initd | 146 ++-- files/fail2ban-tmpfiles.conf | 2 +- files/fail2ban.service.in | 4 +- man/fail2ban-client.1 | 38 +- man/fail2ban-python.1 | 4 +- man/fail2ban-regex.1 | 12 +- man/fail2ban-regex.h2m | 1 + man/fail2ban-server.1 | 4 +- man/fail2ban-testcases.1 | 2 +- man/generate-man | 14 +- man/jail.conf.5 | 43 +- setup.py | 41 +- 134 files changed, 5907 insertions(+), 1373 deletions(-) delete mode 100644 config/action.d/nftables-common.conf create mode 100644 config/action.d/nftables.conf create mode 100644 config/filter.d/bitwarden.conf create mode 100644 config/filter.d/centreon.conf create mode 100644 config/filter.d/traefik-auth.conf create mode 100644 config/filter.d/znc-adminlog.conf create mode 100644 fail2ban/server/observer.py create mode 100644 fail2ban/tests/files/database_v2.db create mode 100644 fail2ban/tests/files/logs/bitwarden create mode 100644 fail2ban/tests/files/logs/centreon create mode 100644 fail2ban/tests/files/logs/sshd-journal create mode 100644 fail2ban/tests/files/logs/traefik-auth create mode 100644 fail2ban/tests/files/logs/znc-adminlog create mode 100644 fail2ban/tests/observertestcase.py diff --git a/.travis.yml b/.travis.yml index ebfcd68e..158cff99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,32 @@ # vim ft=yaml # travis-ci.org definition for Fail2Ban build # https://travis-ci.org/fail2ban/fail2ban/ + +#os: linux + language: python -python: - - 2.6 - - 2.7 - - pypy - # disabled until coverage module fixes up compatibility issue - # - 3.2 - - 3.3 - - 3.4 - - 3.5 - - 3.6 - - 3.7-dev - # disabled since setuptools dropped support for Python 3.0 - 3.2 - # - pypy3 - - pypy3.3-5.5-alpha +dist: xenial + +matrix: + fast_finish: true + include: + - python: 2.6 + dist: trusty # required for Python 2.6 + - python: 2.7 + dist: trusty # required for packages like gamin + name: 2.7 (trusty) + - python: 2.7 + name: 2.7 (xenial) + - python: pypy + dist: trusty + - python: 3.3 + dist: trusty + - python: 3.4 + - python: 3.5 + - python: 3.6 + - python: 3.7 + - python: 3.8-dev + - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi @@ -35,15 +46,22 @@ install: # codecov: - travis_retry pip install codecov # dnspython or dnspython3 - - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3; fi + - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3 || echo 'not installed'; fi + # python systemd bindings: + - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get install -qq python-systemd || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get install -qq python3-systemd || echo 'not installed'; fi # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then (travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/) || echo 'not installed'; fi # pyinotify - - travis_retry pip install pyinotify + - travis_retry pip install pyinotify || echo 'not installed' + # Install helper tools + - sudo apt-get install shellcheck before_script: # Manually execute 2to3 for now - if [[ "$F2B_PY" = 3 ]]; then ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' script: # Keep the legacy setup.py test approach of checking coverage for python2 - if [[ "$F2B_PY" = 2 ]]; then coverage run setup.py test; fi @@ -51,13 +69,14 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux - - test -e /usr/share/doc/fail2ban/FILTERS + # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) + - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi + # Test initd script + - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: - if [[ "$F2B_COV" = 1 ]]; then coveralls; fi - codecov -matrix: - fast_finish: true + # Might be worth looking into #notifications: # email: true diff --git a/ChangeLog b/ChangeLog index f45718d3..cc0c6608 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,53 +6,190 @@ Fail2Ban: Changelog =================== -Incompatibility list (compared to v.0.9): +ver. 0.11.1 (2020/01/11) - this-is-the-way ----------- -* Filter (or `failregex`) internal capture-groups: +### Compatibility: +* to v.0.10: + - 0.11 is totally compatible to 0.10 (configuration- and API-related stuff), but the database + got some new tables and fields (auto-converted during the first start), so once updated to 0.11, you + have to remove the database /var/lib/fail2ban/fail2ban.sqlite3 (or its different to 0.10 schema) + if you would need to downgrade to 0.10 for some reason. +* to v.0.9: + - Filter (or `failregex`) internal capture-groups: - - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should - rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)` - (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings). + * If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should + rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)` + (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings). - Of course you can always define your own capture-group (like below `_cond_ip_`) to do this. - ``` - testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1" - fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_>): bad host (?P=_cond_ip_)$" - ``` - - New internal groups (currently reserved for internal usage): - `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if - mapping from tag `` used in failregex (e. g. `user` by ``). + Of course you can always define your own capture-group (like below `_cond_ip_`) to do this. + ``` + testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1" + fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_>): bad host (?P=_cond_ip_)$" + ``` + * New internal groups (currently reserved for internal usage): + `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if + mapping from tag `` used in failregex (e. g. `user` by ``). -* v.0.10 uses more precise date template handling, that can be theoretically incompatible to some - user configurations resp. `datepattern`. + - v.0.10 and 0.11 use more precise date template handling, that can be theoretically incompatible to some + user configurations resp. `datepattern`. -* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are - IPv6-capable now. - - -ver. 0.10.5-dev-1 (20??/??/??) - development edition ------------ + - Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are + IPv6-capable now. ### Fixes +* purge database will be executed now (within observer). +* restoring currently banned ip after service restart fixed + (now < timeofban + bantime), ignore old log failures (already banned) +* upgrade database: update new created table `bips` with entries from table `bans` (allows restore + current bans after upgrade from version <= 0.10) + +### New Features +* Increment ban time (+ observer) functionality introduced. +* Database functionality extended with bad ips. +* New tags (usable in actions): + - `` - ban count of this offender if known as bad (started by 1 for unknown) + - `` - current ban-time of the ticket (prolongation can be retarded up to 10 sec.) +* Introduced new action command `actionprolong` to prolong ban-time (e. g. set new timeout if expected); + Several actions (like ipset, etc.) rewritten using net logic with `actionprolong`. + Note: because ban-time is dynamic, it was removed from jail.conf as timeout argument (check jail.local). + +### Enhancements +* algorithm of restore current bans after restart changed: update the restored ban-time (and therefore + end of ban) of the ticket with ban-time of jail (as maximum), for all tickets with ban-time greater + (or persistent); not affected if ban-time of the jail is unchanged between stop/start. +* added new setup-option `--without-tests` to skip building and installing of tests files (gh-2287). +* added new command `fail2ban-client get banip ?sep-char|--with-time?` to get the banned ip addresses (gh-1916). + + +ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must +----------- + +Yes, Hrrrm... + +### Fixes +* [compatibility] systemd backend: default flags changed to SYSTEM_ONLY(4), fixed in gh-2444 in order to ignore + user session files per default, so could prevent "Too many open files" errors on a lot of user sessions (see gh-2392) +* [grave] fixed parsing of multi-line filters (`maxlines` > 1) together with systemd backend, + now systemd-filter replaces newlines in message from systemd journal with `\n` (otherwise + multi-line parsing may be broken, because removal of matched string from multi-line buffer window + is confused by such extra new-lines, so they are retained and got matched on every followed + message, see gh-2431) +* [stability] prevent race condition - no unban if the bans occur continuously (gh-2410); + now an unban-check will happen not later than 10 tickets get banned regardless there are + still active bans available (precedence of ban over unban-check is 10 now) * fixed read of included config-files (`.local` overwrites options of `.conf` for config-files included with before/after) +* `action.d/abuseipdb.conf`: switched to use AbuseIPDB API v2 (gh-2302) +* `action.d/badips.py`: fixed start of banaction on demand (which may be IP-family related), gh-2390 +* `action.d/helpers-common.conf`: rewritten grep arguments, now options `-wF` used to match only + whole words and fixed string (not as pattern), gh-2298 +* `filter.d/apache-auth.conf`: + - ignore errors from mod_evasive in `normal` mode (mode-controlled now) (gh-2548); + - extended with option `mode` - `normal` (default) and `aggressive` * `filter.d/sshd.conf`: + - matches `Bad protocol version identification` in `ddos` and `aggressive` modes (gh-2404). - captures `Disconnecting ...: Change of username or service not allowed` (gh-2239, gh-2279) - - captures `Disconnected from ... [preauth]` (`extra`/`aggressive` mode and preauth phase only, gh-2239, gh-2279) + - captures `Disconnected from ... [preauth]`, preauth phase only, different handling by `extra` + (with supplied user only) and `ddos`/`aggressive` mode (gh-2115, gh-2239, gh-2279) * `filter.d/mysqld-auth.conf`: - MYSQL 8.0.13 compatibility (log-error-verbosity = 3), log-format contains few additional words enclosed in brackets after "[Note]" (gh-2314) +* `filter.d/sendmail-reject.conf`: + - `mode=extra` now captures port IDs of `TLSMTA` and `MSA` (defaults for ports 465 and 587 on some distros) * `files/fail2ban.service.in`: fixed systemd-unit template - missing nftables dependency (gh-2313) * several `action.d/mail*`: fixed usage with multiple log files (ultimate fix for gh-976, gh-2341) +* `filter.d/sendmail-reject.conf`: fixed journal usage for some systems (e. g. CentOS): if only identifier + set to `sm-mta` (no unit `sendmail`) for some messages (gh-2385) +* `filter.d/asterisk.conf`: asterisk can log additional timestamp if logs into systemd-journal + (regex extended with optional part matching this, gh-2383) +* `filter.d/postfix.conf`: + - regexp's accept variable suffix code in status of postfix for precise messages (gh-2442) + - extended with new postfix filter mode `errors` to match "too many errors" (gh-2439), + also included within modes `normal`, `more` (`extra` and `aggressive`), since postfix + parameter `smtpd_hard_error_limit` is default 20 (additionally consider `maxretry`) +* `filter.d/named-refused.conf`: + - support BIND 9.11.0 log format (includes an additional field @0xXXX..., gh-2406); + - `prefregex` extended, more selective now (denied/NOTAUTH suffix moved from failregex, so no catch-all there anymore) +* `filter.d/sendmail-auth.conf`, `filter.d/sendmail-reject.conf` : + - ID in prefix can be longer as 14 characters (gh-2563); +* all filters would accept square brackets around IPv4 addresses also (e. g. monit-filter, gh-2494) +* avoids unhandled exception during flush (gh-2588) +* fixes pass2allow-ftp jail - due to inverted handling, action should prohibit access per default for any IP, + therefore reset start on demand parameter for this action (it will be started immediately by repair); +* auto-detection of IPv6 subsystem availability (important for not on-demand actions or jails, like pass2allow); ### New Features +* new replacement tags for failregex to match subnets in form of IP-addresses with CIDR mask (gh-2559): + - `` - helper regex to match CIDR (simple integer form of net-mask); + - `` - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional); +* grouped tags (``, ``, ``) recognize IP addresses enclosed in square brackets * new failregex-flag tag `` for failregex, signaled that the access to service was gained (ATM used similar to tag ``, but it does not add the log-line to matches, gh-2279) +* filters: introduced new configuration parameter `logtype` (default `file` for file-backends, and + `journal` for journal-backends, gh-2387); can be also set to `rfc5424` to force filters (which include common.conf) + to use RFC 5424 conform prefix-line per default (gh-2467); +* for better performance and safety the option `logtype` can be also used to + select short prefix-line for file-backends too for all filters using `__prefix_line` (`common.conf`), + if message logged only with `hostname svc[nnnn]` prefix (often the case on several systems): +```ini +[jail] +backend = auto +filter = flt[logtype=short] +``` +* `filter.d/common.conf`: differentiate `__prefix_line` for file/journal logtype's (speedup and fix parsing + of systemd-journal); +* `filter.d/traefik-auth.conf`: used to ban hosts, that were failed through traefik +* `filter.d/znc-adminlog.conf`: new filter for ZNC (IRC bouncer); requires the adminlog module to be loaded ### Enhancements +* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to contol + how many matches per ticket fail2ban can hold in memory and store in database (gh-2402, gh-2118); +* fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size + of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to + avoid runtime error "can't start new thread" (see gh-969); * jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations containing new-line); +* fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349); + Syntax: + - `fail2ban-client set banip ... ` + - `fail2ban-client set unbanip [--report-absent] ... ` +* fail2ban-client: extended with new feature which allows to inform fail2ban about single or multiple + attempts (failure) for IP (resp. failure-ID), see gh-2351; + Syntax: + - `fail2ban-client set attempt [ ... ]` +* `action.d/nftables.conf`: + - isolate fail2ban rules into a dedicated table and chain (gh-2254) + - `nftables-allports` supports multiple protocols in single rule now + - combined nftables actions to single action `nftables`: + * `nftables-common` is removed (replaced with single action `nftables` now) + * `nftables-allports` is obsolete, superseded by `nftables[type=allports]` + * `nftables-multiport` is obsolete, superseded by `nftables[type=multiport]` + - allowed multiple protocols in `nftables[type=multiport]` action (single set with multiple rules + in chain), following configuration in jail would replace 3 separate actions, see + https://github.com/fail2ban/fail2ban/pull/2254#issuecomment-534684675 +* `action.d/badips.py`: option `loglevel` extended with level of summary message, + following example configuration logging summary with NOTICE and rest with DEBUG log-levels: + `action = badips.py[loglevel="debug, notice"]` +* samplestestcase.py (testSampleRegexsFactory) extended: + - allow coverage of journal logtype; + - new option `fileOptions` to set common filter/test options for whole test-file; +* large enhancement: auto-reban, improved invariant check and conditional operations (gh-2588): + - improves invariant check and repair (avoid unhandled exception, consider family on conditional operations, etc), + prepared for bulk re-ban in repair case (if bulk-ban becomes implemented); + - automatic reban (repeat banning action) after repair/restore sane environment, if already logged ticket causes + new failures (via new action operation `actionreban` or `actionban` if still not defined in action); + * introduces banning epoch for actions and tickets (to distinguish or recognize removed set of the tickets); + * invariant check avoids repair by unban/stop (unless parameter `actionrepair_on_unban` set to `true`); + * better handling for all conditional operations (distinguish families for certain operations like + repair/flush/stop, prepared for other families, e. g. if different handling for subnets expected, etc); + * partially implements gh-980 (more breakdown safe handling); + * closes gh-1680 (better as large-scale banning implementation with on-demand reban by failure, + at least unless a bulk-ban gets implemented); +* fail2ban-regex - several enhancements and fixes: + - improved usage output (don't put a long help if an error occurs); + - new option `--no-check-all` to avoid check of all regex's (first matched only); + - new option `-o`, `--out` to set token only provided in output (disables check-all and outputs only expected data). ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four @@ -303,9 +440,14 @@ TODO: implementing of options resp. other tasks from PR #1346 - `` - failure identifier (if raw resp. failures without IP address) - `` - PTR reversed representation of IP address - `` - host name of the IP address + - `` - ban count of this offender if known as bad (started by 1 for unknown) + - `` - current ban-time of the ticket (prolongation can be retarded up to 10 sec.) - `` - interpolates to the corresponding filter group capture `...` - `` - fully-qualified name of host (the same as `$(hostname -f)`) - `` - short hostname (the same as `$(uname -n)`) +* Introduced new action command `actionprolong` to prolong ban-time (e. g. set new timeout if expected); + Several actions (like ipset, etc.) rewritten using net logic with `actionprolong`. + Note: because ban-time is dynamic, it was removed from jail.conf as timeout argument (check jail.local). * Allow to use filter options by `fail2ban-regex`, example: fail2ban-regex text.log "sshd[mode=aggressive]" * Samples test case factory extended with filter options - dict in JSON to control @@ -379,6 +521,9 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc * testSocket: sporadical bug repaired - wait for server thread starts a socket (listener) * testExecuteTimeoutWithNastyChildren: sporadical bug repaired - wait for pid file inside bash, kill tree in any case (gh-1155) +* purge database will be executed now (within observer). +* restoring currently banned ip after service restart fixed + (now < timeofban + bantime), ignore old log failures (already banned) * Fixed high-load of pyinotify-backend, see https://github.com/fail2ban/fail2ban/issues/885#issuecomment-248964591 * Database: stability fix - repack cursor iterator as long as locked @@ -416,6 +561,9 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc - new conditional section functionality used in config resp. includes: - [Init?family=inet4] - IPv4 qualified hosts only - [Init?family=inet6] - IPv6 qualified hosts only +* Increment ban time (+ observer) functionality introduced. + Thanks Serg G. Brester (sebres) +* Database functionality extended with bad ips. * New reload functionality (now totally without restart, unbanning/rebanning, etc.), see gh-1557 * Several commands extended and new commands introduced: diff --git a/MANIFEST b/MANIFEST index 144ae697..3974184c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -42,7 +42,7 @@ config/action.d/mail-whois-lines.conf config/action.d/mynetwatchman.conf config/action.d/netscaler.conf config/action.d/nftables-allports.conf -config/action.d/nftables-common.conf +config/action.d/nftables.conf config/action.d/nftables-multiport.conf config/action.d/nginx-block-map.conf config/action.d/npf.conf @@ -81,7 +81,9 @@ config/filter.d/apache-pass.conf config/filter.d/apache-shellshock.conf config/filter.d/assp.conf config/filter.d/asterisk.conf +config/filter.d/bitwarden.conf config/filter.d/botsearch-common.conf +config/filter.d/centreon.conf config/filter.d/common.conf config/filter.d/counter-strike.conf config/filter.d/courier-auth.conf @@ -145,11 +147,13 @@ config/filter.d/sshd.conf config/filter.d/stunnel.conf config/filter.d/suhosin.conf config/filter.d/tine20.conf +config/filter.d/traefik-auth.conf config/filter.d/uwimap-auth.conf config/filter.d/vsftpd.conf config/filter.d/webmin-auth.conf config/filter.d/wuftpd.conf config/filter.d/xinetd-fail.conf +config/filter.d/znc-adminlog.conf config/filter.d/zoneminder.conf config/jail.conf config/paths-arch.conf @@ -203,6 +207,7 @@ fail2ban/server/jail.py fail2ban/server/jails.py fail2ban/server/jailthread.py fail2ban/server/mytime.py +fail2ban/server/observer.py fail2ban/server/server.py fail2ban/server/strptime.py fail2ban/server/ticket.py @@ -219,6 +224,7 @@ fail2ban/tests/actiontestcase.py fail2ban/tests/banmanagertestcase.py fail2ban/tests/clientbeautifiertestcase.py fail2ban/tests/clientreadertestcase.py +fail2ban/tests/config/action.d/action.conf fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf fail2ban/tests/config/filter.d/simple.conf @@ -256,6 +262,7 @@ fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htpasswd fail2ban/tests/files/config/apache-auth/noentry/.htaccess fail2ban/tests/files/config/apache-auth/README fail2ban/tests/files/database_v1.db +fail2ban/tests/files/database_v2.db fail2ban/tests/files/filter.d/substition.conf fail2ban/tests/files/filter.d/testcase01.conf fail2ban/tests/files/filter.d/testcase-common.conf @@ -273,9 +280,11 @@ fail2ban/tests/files/logs/apache-pass fail2ban/tests/files/logs/apache-shellshock fail2ban/tests/files/logs/assp fail2ban/tests/files/logs/asterisk +fail2ban/tests/files/logs/bitwarden fail2ban/tests/files/logs/bsd/syslog-plain.txt fail2ban/tests/files/logs/bsd/syslog-v.txt fail2ban/tests/files/logs/bsd/syslog-vv.txt +fail2ban/tests/files/logs/centreon fail2ban/tests/files/logs/counter-strike fail2ban/tests/files/logs/courier-auth fail2ban/tests/files/logs/courier-smtp @@ -332,14 +341,17 @@ fail2ban/tests/files/logs/solid-pop3d fail2ban/tests/files/logs/squid fail2ban/tests/files/logs/squirrelmail fail2ban/tests/files/logs/sshd +fail2ban/tests/files/logs/sshd-journal fail2ban/tests/files/logs/stunnel fail2ban/tests/files/logs/suhosin fail2ban/tests/files/logs/tine20 +fail2ban/tests/files/logs/traefik-auth fail2ban/tests/files/logs/uwimap-auth fail2ban/tests/files/logs/vsftpd fail2ban/tests/files/logs/webmin-auth fail2ban/tests/files/logs/wuftpd fail2ban/tests/files/logs/xinetd-fail +fail2ban/tests/files/logs/znc-adminlog fail2ban/tests/files/logs/zoneminder fail2ban/tests/files/logs/zzz-generic-example fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline @@ -356,6 +368,7 @@ fail2ban/tests/files/zzz-sshd-obsolete-multiline.log fail2ban/tests/filtertestcase.py fail2ban/tests/__init__.py fail2ban/tests/misctestcase.py +fail2ban/tests/observertestcase.py fail2ban/tests/samplestestcase.py fail2ban/tests/servertestcase.py fail2ban/tests/sockettestcase.py diff --git a/README.md b/README.md index 2b40d306..8e9f5c3a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.10.3.dev1 20??/??/?? + v0.11.0.dev1 20??/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -18,11 +18,12 @@ attempts, it cannot eliminate the risk presented by weak authentication. Set up services to use only two factor, or public/private authentication mechanisms if you really want to protect services. - | Since v0.10 fail2ban supports the matching of the IPv6 addresses. + | Since v0.10 fail2ban supports the matching of IPv6 addresses. ------|------ This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs -to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki) +to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki), +[Developers documentation](https://fail2ban.readthedocs.io/) and the website: https://www.fail2ban.org Installation: @@ -45,10 +46,16 @@ Optional: To install: - tar xvfj fail2ban-0.10.3.tar.bz2 - cd fail2ban-0.10.3 - python setup.py install + tar xvfj fail2ban-0.11.0.tar.bz2 + cd fail2ban-0.11.0 + sudo python setup.py install + +Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, 0.11 + git clone https://github.com/fail2ban/fail2ban.git + cd fail2ban + sudo python setup.py install + This will install Fail2Ban into the python library directory. The executable scripts are placed into `/usr/bin`, and configuration in `/etc/fail2ban`. @@ -58,6 +65,9 @@ Fail2Ban should be correctly installed now. Just type: to see if everything is alright. You should always use fail2ban-client and never call fail2ban-server directly. +You can verify that you have the correct version installed with + + fail2ban-client version Please note that the system init/service script is not automatically installed. To enable fail2ban as an automatic service, simply copy the script for your @@ -79,11 +89,11 @@ fail2ban(1) and jail.conf(5) manpages for further references. Code status: ------------ -* [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) travis-ci.org (0.10 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.png?branch=master)](https://travis-ci.org/fail2ban/fail2ban) travis-ci.org (master branch) +* travis-ci.org: [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.11)](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) -* [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) +* coveralls.io: [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.11)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) -* [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) +* codecov.io: [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.11)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) Contact: -------- diff --git a/THANKS b/THANKS index 7861ceb5..c363c76c 100644 --- a/THANKS +++ b/THANKS @@ -111,7 +111,7 @@ Russell Odom SATO Kentaro Sean DuBois Sebastian Arcus -Serg G. Brester +Serg G. Brester (sebres) Sergey Safarov Shaun C. Sireyessire diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index c53ed489..010af5b5 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -47,6 +47,9 @@ [Definition] +# bypass action for restored tickets +norestored = 1 + # Option: actionstart # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD @@ -80,13 +83,10 @@ actioncheck = # wherever you install the helper script. For the PHP helper script, see # # -# --ciphers ecdhe_ecdsa_aes_256_sha is used to workaround a -# "NSS error -12286" from curl as it attempts to connect using -# SSLv3. See https://www.centos.org/forums/viewtopic.php?t=52732 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = lgm=$(printf '%%s\n...' ""); curl --fail --tlsv1.1 --data "key=" --data-urlencode "comment=$lgm" --data "ip=" --data "category=" "https://www.abuseipdb.com/report/json" +actionban = lgm=$(printf '%%.1000s\n...' ""); curl -sSf "https://api.abuseipdb.com/api/v2/report" -H "Accept: application/json" -H "Key: " --data-urlencode "comment=$lgm" --data-urlencode "ip=" --data "categories=" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -101,5 +101,5 @@ actionunban = # Notes Your API key from abuseipdb.com # Values: STRING Default: None # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. -# You will need to set the catagory in the action call. +# You will need to set the category in the action call. abuseipdb_apikey = diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 1ad711f4..805120e9 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -31,8 +31,8 @@ else: # pragma: 3.x no cover from urllib2 import Request, urlopen, HTTPError from urllib import urlencode -from fail2ban.server.actions import ActionBase -from fail2ban.helpers import str2LogLevel +from fail2ban.server.actions import Actions, ActionBase, BanTicket +from fail2ban.helpers import splitwords, str2LogLevel @@ -54,9 +54,6 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable age : str, optional Age of last report for bad IPs, per badips.com syntax. Default "24h" (24 hours) - key : str, optional - Key issued by badips.com to report bans, for later retrieval - of personalised content. banaction : str, optional Name of banaction to use for blacklisting bad IPs. If `None`, no blacklist of IPs will take place. @@ -67,14 +64,17 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable "postfix", but want to use whole "mail" category for blacklist. Default `category`. bankey : str, optional - Key issued by badips.com to blacklist IPs reported with the - associated key. + Key issued by badips.com to retrieve personal list + of blacklist IPs. updateperiod : int, optional Time in seconds between updating bad IPs blacklist. Default 900 (15 minutes) loglevel : int/str, optional Log level of the message when an IP is (un)banned. Default `DEBUG`. + Can be also supplied as two-value list (comma- or space separated) to + provide level of the summary message when a group of IPs is (un)banned. + Example `DEBUG,INFO`. agent : str, optional User agent transmitted to server. Default `Fail2Ban/ver.` @@ -90,9 +90,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable def _Request(self, url, **argv): return Request(url, headers={'User-Agent': self.agent}, **argv) - def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900, loglevel='DEBUG', agent="Fail2Ban", - timeout=TIMEOUT): + def __init__(self, jail, name, category, score=3, age="24h", + banaction=None, bancategory=None, bankey=None, updateperiod=900, + loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): super(BadIPsAction, self).__init__(jail, name) self.timeout = timeout @@ -100,11 +100,12 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable self.category = category self.score = score self.age = age - self.key = key self.banaction = banaction self.bancategory = bancategory or category self.bankey = bankey - self.loglevel = str2LogLevel(loglevel) + loglevel = splitwords(loglevel) + self.sumloglevel = str2LogLevel(loglevel[-1]) + self.loglevel = str2LogLevel(loglevel[0]) self.updateperiod = updateperiod self._bannedips = set() @@ -281,13 +282,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable def _banIPs(self, ips): for ip in ips: try: - self._jail.actions[self.banaction].ban({ - 'ip': ip, - 'failures': 0, - 'matches': "", - 'ipmatches': "", - 'ipjailmatches': "", - }) + ai = Actions.ActionInfo(BanTicket(ip), self._jail) + self._jail.actions[self.banaction].ban(ai) except Exception as e: self._logSys.error( "Error banning IP %s for jail '%s' with action '%s': %s", @@ -302,13 +298,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable def _unbanIPs(self, ips): for ip in ips: try: - self._jail.actions[self.banaction].unban({ - 'ip': ip, - 'failures': 0, - 'matches': "", - 'ipmatches': "", - 'ipjailmatches': "", - }) + ai = Actions.ActionInfo(BanTicket(ip), self._jail) + self._jail.actions[self.banaction].unban(ai) except Exception as e: self._logSys.error( "Error unbanning IP %s for jail '%s' with action '%s': %s", @@ -350,9 +341,13 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable s = ips - self._bannedips p = len(s) self._banIPs(s) - self._logSys.log(self.loglevel, - "Updated IPs for jail '%s' (-%d/+%d). Update again in %i seconds", - self._jail.name, m, p, self.updateperiod) + if m != 0 or p != 0: + self._logSys.log(self.sumloglevel, + "Updated IPs for jail '%s' (-%d/+%d)", + self._jail.name, m, p) + self._logSys.debug( + "Next update for jail '%' in %i seconds", + self._jail.name, self.updateperiod) finally: self._timer = threading.Timer(self.updateperiod, self.update) self._timer.start() @@ -382,8 +377,6 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable """ try: url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - if self.key: - url = "?".join([url, urlencode({'key': self.key})]) self._logSys.debug('badips.com: ban, url: %r', url) response = urlopen(self._Request(url), timeout=self.timeout) except HTTPError as response: # pragma: no cover diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index dcf20375..a1065224 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -29,6 +29,8 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 timeout -exist +actionprolong = %(actionban)s + actionunban = ipset del -exist [Init] @@ -40,11 +42,11 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 600 -bantime = 600 +default-timeout = 600 # Option: actiontype # Notes.: defines additions to the blocking rule diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf index b036f68f..03422a87 100644 --- a/config/action.d/helpers-common.conf +++ b/config/action.d/helpers-common.conf @@ -4,8 +4,9 @@ # _grep_logs_args = 'test' # (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... # -_grep_logs = logpath=""; grep -E %(_grep_logs_args)s $logpath | -_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)" +_grep_logs = logpath=""; grep %(_grep_logs_args)s $logpath | +# options `-wF` used to match only whole words and fixed string (not as pattern) +_grep_logs_args = -wF "" # Used for actions, that should not by executed if ticket was restored: _bypass_if_restored = if [ '' = '1' ]; then exit 0; fi; diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index dc7d63a7..c851233c 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -51,6 +51,8 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionprolong = %(actionban)s + # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the # command is executed with Fail2Ban user rights. @@ -61,11 +63,11 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 600 -# -bantime = 600 + +default-timeout = 600 ipmset = f2b- familyopt = diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index f88777b8..12c3ddd6 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -51,6 +51,8 @@ actionstop = -D -p -m multiport --dports -m # actionban = ipset add timeout -exist +actionprolong = %(actionban)s + # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the # command is executed with Fail2Ban user rights. @@ -61,11 +63,11 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 600 -# -bantime = 600 + +default-timeout = 600 ipmset = f2b- familyopt = diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index b0d27afc..ecf3a5d9 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -17,7 +17,7 @@ _whois = whois || echo "missing whois program" # character set before sending it to a mail program # make sure you have 'file' and 'iconv' commands installed when opting for that _whois_target_charset = UTF-8 -_whois_convert_charset = whois | +_whois_convert_charset = (%(_whois)s) | { WHOIS_OUTPUT=$(cat) ; WHOIS_CHARSET=$(printf %%b "$WHOIS_OUTPUT" | file -b --mime-encoding -) ; printf %%b "$WHOIS_OUTPUT" | iconv -f $WHOIS_CHARSET -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf index 6c69da39..908abe40 100644 --- a/config/action.d/nftables-allports.conf +++ b/config/action.d/nftables-allports.conf @@ -6,17 +6,12 @@ # Modified: Alexander Belykh # adapted for nftables # +# Obsolete: superseded by nftables[type=allports] [INCLUDES] -before = nftables-common.conf +before = nftables.conf [Definition] -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = meta l4proto - -[Init] +type = allports diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf deleted file mode 100644 index 37045712..00000000 --- a/config/action.d/nftables-common.conf +++ /dev/null @@ -1,135 +0,0 @@ -# Fail2Ban configuration file -# -# Author: Daniel Black -# Author: Cyril Jaquier -# Modified: Yaroslav O. Halchenko -# made active on all ports from original iptables.conf -# Modified: Alexander Belykh -# adapted for nftables -# -# This is a included configuration file and includes the definitions for the nftables -# used in all nftables based actions by default. -# -# The user can override the defaults in nftables-common.local - -[INCLUDES] - -after = nftables-common.local - -[Definition] - -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = dport \{ \} - -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = add set \{ type \; \} - insert rule %(nftables_mode)s saddr @ - -_nft_list = --handle --numeric list chain -_nft_get_handle_id = grep -m1 ' saddr @ # handle' | grep -oe ' handle [0-9]*' - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s) - delete rule $HANDLE_ID - delete set - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = list chain | grep -q '@[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = add element \{ \} - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = delete element \{ \} - -[Init] - -# Option: nftables_type -# Notes.: address type to work with -# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr -# -nftables_type = ipv4_addr - -# Option: nftables_family -# Notes.: address family to work in -# Values: [ip | ip6 | inet] Default: inet -# -nftables_family = inet - -# Option: nftables_table -# Notes.: table in the address family to work in -# Values: STRING Default: filter -# -nftables_table = filter - -# Option: chain -# Notes specifies the nftables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: input -chain = input - -# Default name of the filtering set -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp ] Default: tcp -# -protocol = tcp - -# Option: blocktype -# Note: This is what the action does with rules. This can be any jump target -# as per the nftables man page (section 8). Common values are drop -# reject, reject with icmp type host-unreachable -# Values: STRING -blocktype = reject - -# Option: nftables -# Notes.: Actual command to be executed, including common to all calls options -# Values: STRING -nftables = nft - -# Option: set_name -# Notes.: The name of the nft set used to store banned addresses -# Values: STRING -set_name = f2b- - -# Option: address_family -# Notes.: The family of the banned addresses -# Values: [ ip | ip6 ] -address_family = ip - -[Init?family=inet6] - -nftables_type = ipv6_addr -set_name = f2b-6 -address_family = ip6 diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index d1afafb3..ba3ec92c 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -6,17 +6,12 @@ # Modified: Alexander Belykh # adapted for nftables # +# Obsolete: superseded by nftables[type=multiport] [INCLUDES] -before = nftables-common.conf +before = nftables.conf [Definition] -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = dport \{ \} - -[Init] +type = multiport \ No newline at end of file diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf new file mode 100644 index 00000000..c1fb8550 --- /dev/null +++ b/config/action.d/nftables.conf @@ -0,0 +1,203 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf +# Modified: Alexander Belykh +# adapted for nftables +# +# This is a included configuration file and includes the definitions for the nftables +# used in all nftables based actions by default. +# +# The user can override the defaults in nftables-common.local +# Example: redirect flow to honeypot +# +# [Init] +# table_family = ip +# chain_type = nat +# chain_hook = prerouting +# chain_priority = -50 +# blocktype = counter redirect to 2222 + +[INCLUDES] + +after = nftables-common.local + +[Definition] + +# Option: type +# Notes.: type of the action. +# Values: [ multiport | allports ] Default: multiport +# +type = multiport + +rule_match-custom = +rule_match-allports = meta l4proto \{ \} +rule_match-multiport = $proto dport \{ \} +match = > + +# Option: rule_stat +# Notes.: statement for nftables filter rule. +# leaving it empty will block all (include udp and icmp) +# Values: nftables statement +# +rule_stat = %(match)s saddr @ + +# optional interator over protocol's: +_nft_for_proto-custom-iter = +_nft_for_proto-custom-done = +_nft_for_proto-allports-iter = +_nft_for_proto-allports-done = +_nft_for_proto-multiport-iter = for proto in $(echo '' | sed 's/,/ /g'); do +_nft_for_proto-multiport-done = done + +_nft_list = -a list chain +_nft_get_handle_id = grep -oP '@\s+.*\s+\Khandle\s+(\d+)$' + +_nft_add_set = add set
\{ type \; \} + <_nft_for_proto--iter> + add rule
%(rule_stat)s + <_nft_for_proto--done> +_nft_del_set = { %(_nft_list)s | %(_nft_get_handle_id)s; } | while read -r hdl; do + delete rule
$hdl; done + delete set
+ +# Option: _nft_shutdown_table +# Notes.: command executed after the stop in order to delete table (it checks that no sets are available): +# Values: CMD +# +_nft_shutdown_table = { list table
| grep -qP '^\s+set\s+'; } || { + delete table
+ } + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = add table
+ -- add chain
\{ type hook priority \; \} + %(_nft_add_set)s + +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action); +# uses `nft flush set ...` and as fallback (e. g. unsupported) recreates the set (with references) +# Values: CMD +# +actionflush = { flush set
2> /dev/null; } || { + %(_nft_del_set)s + %(_nft_add_set)s + } + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = %(_nft_del_set)s + <_nft_shutdown_table> + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain
| grep -q '@[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element
\{ \} + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element
\{ \} + +[Init] + +# Option: table +# Notes.: main table to store chain and sets (automatically created on demand) +# Values: STRING Default: f2b-table +table = f2b-table + +# Option: table_family +# Notes.: address family to work in +# Values: [ip | ip6 | inet] Default: inet +table_family = inet + +# Option: chain +# Notes.: main chain to store rules +# Values: STRING Default: f2b-chain +chain = f2b-chain + +# Option: chain_type +# Notes.: refers to the kind of chain to be created +# Values: [filter | route | nat] Default: filter +# +chain_type = filter + +# Option: chain_hook +# Notes.: refers to the kind of chain to be created +# Values: [ prerouting | input | forward | output | postrouting ] Default: input +# +chain_hook = input + +# Option: chain_priority +# Notes.: priority in the chain. +# Values: NUMBER Default: -1 +# +chain_priority = -1 + +# Option: addr_type +# Notes.: address type to work with +# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr +# +addr_type = ipv4_addr + +# Default name of the filtering set +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the nftables man page (section 8). Common values are drop, +# reject, reject with icmpx type host-unreachable, redirect to 2222 +# Values: STRING +blocktype = reject + +# Option: nftables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +nftables = nft + +# Option: addr_set +# Notes.: The name of the nft set used to store banned addresses +# Values: STRING +addr_set = addr-set- + +# Option: addr_family +# Notes.: The family of the banned addresses +# Values: [ ip | ip6 ] +addr_family = ip + +[Init?family=inet6] +addr_family = ip6 +addr_type = ipv6_addr +addr_set = addr6-set- diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index 33c15f9c..0b6aa0ad 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -105,4 +105,4 @@ actioncheck = actionban = echo "\\\\ 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s -actionunban = id=$(echo "" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/$id 1;/d" %(blck_lst_file)s; %(blck_lst_reload)s +actionunban = id=$(echo "" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^\\\\$id 1;$/d" %(blck_lst_file)s; %(blck_lst_reload)s diff --git a/config/action.d/osx-afctl.conf b/config/action.d/osx-afctl.conf index a319fc6b..a75e5723 100644 --- a/config/action.d/osx-afctl.conf +++ b/config/action.d/osx-afctl.conf @@ -12,5 +12,5 @@ actioncheck = actionban = /usr/libexec/afctl -a -t actionunban = /usr/libexec/afctl -r -[Init] -bantime = 2880 +actionprolong = %(actionunban)s && %(actionban)s + diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf index 199c6ce5..13803f8b 100644 --- a/config/action.d/sendmail-buffered.conf +++ b/config/action.d/sendmail-buffered.conf @@ -24,7 +24,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on The jail has been started successfully.\n Output will be buffered until lines are available.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -38,7 +38,7 @@ actionstop = if [ -f ]; then These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | rm fi printf %%b "Subject: [Fail2Ban] : stopped on @@ -47,7 +47,7 @@ actionstop = if [ -f ]; then Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | # Option: actioncheck # Notes.: command executed once before each actionban command @@ -71,7 +71,7 @@ actionban = printf %%b "`date`: ( failures)\n" >> These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | rm fi diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf index 9bf15054..1e31fadf 100644 --- a/config/action.d/sendmail-common.conf +++ b/config/action.d/sendmail-common.conf @@ -21,7 +21,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] : started on Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -34,7 +34,7 @@ actionstop = printf %%b "Subject: [Fail2Ban] : stopped on Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | # Option: actioncheck # Notes.: command executed once before each actionban command @@ -60,6 +60,10 @@ actionunban = [Init] +# Your system mail command +# +mailcmd = /usr/sbin/sendmail -f "" "" + # Recipient mail address # dest = root diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index b7c1bf36..b36e49a7 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -37,11 +37,11 @@ actionban = ( printf %%b "Subject: [Fail2Ban] : banned from " | cut -d':' -f2-` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "" | cut -d':' -f2-` hostname: \n\n - Lines containing failures of \n"; + Lines containing failures of (max )\n"; %(_grep_logs)s; printf %%b "\n Regards,\n - Fail2Ban" ) | /usr/sbin/sendmail -f + Fail2Ban" ) | [Init] diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 06ea3a3e..7790ec53 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois `\n\n + `%(_whois_command)s`\n\n Matches for with failures IP:\n \n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | [Init] diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index 83bff1b4..e4717ca1 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois `\n\n + `%(_whois_command)s`\n\n Matches with failures IP:\n \n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | [Init] diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4b947cb2..47ec6ed5 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf helpers-common.conf [Definition] @@ -27,13 +28,13 @@ actionban = ( printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n - `/usr/bin/whois || echo missing whois program`\n\n - Lines containing failures of \n"; + Here is more information about :\n" + %(_whois_command)s; + printf %%b "\nLines containing failures of (max )\n"; %(_grep_logs)s; printf %%b "\n Regards,\n - Fail2Ban" ) | /usr/sbin/sendmail -f + Fail2Ban" ) | [Init] diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index 01520135..08215ea7 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois `\n\n + `%(_whois_command)s`\n\n Matches:\n \n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | [Init] diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index 2fb01ed3..9e93cd32 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,9 +28,9 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `/usr/bin/whois || echo missing whois program`\n + `%(_whois_command)s`\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | [Init] diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index cf420915..ad9e8d79 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -27,7 +27,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from has just been banned by Fail2Ban after attempts against .\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" | [Init] diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index fc7dd24e..45be0c0a 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -68,6 +68,8 @@ actionstop = ipset flush f2b- # actionban = ipset add f2b- timeout -exist +actionprolong = %(actionban)s + # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the # command is executed with Fail2Ban user rights. @@ -76,10 +78,8 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -[Init] - -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 600 -# -bantime = 600 + +default-timeout = 600 diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py index 9cdfe327..5c27d0ff 100644 --- a/config/action.d/smtp.py +++ b/config/action.d/smtp.py @@ -159,25 +159,25 @@ class SMTPAction(ActionBase): try: self._logSys.debug("Connected to SMTP '%s', response: %i: %s", self.host, *smtp.connect(self.host)) - if self.user and self.password: + if self.user and self.password: # pragma: no cover (ATM no tests covering that) smtp.login(self.user, self.password) failed_recipients = smtp.sendmail( self.fromaddr, self.toaddr.split(", "), msg.as_string()) - except smtplib.SMTPConnectError: + except smtplib.SMTPConnectError: # pragma: no cover self._logSys.error("Error connecting to host '%s'", self.host) raise - except smtplib.SMTPAuthenticationError: + except smtplib.SMTPAuthenticationError: # pragma: no cover self._logSys.error( "Failed to authenticate with host '%s' user '%s'", self.host, self.user) raise - except smtplib.SMTPException: + except smtplib.SMTPException: # pragma: no cover self._logSys.error( "Error sending mail to host '%s' from '%s' to '%s'", self.host, self.fromaddr, self.toaddr) raise else: - if failed_recipients: + if failed_recipients: # pragma: no cover self._logSys.warning( "Email to '%s' failed to following recipients: %r", self.toaddr, failed_recipients) @@ -186,7 +186,7 @@ class SMTPAction(ActionBase): try: self._logSys.debug("Disconnected from '%s', response %i: %s", self.host, *smtp.quit()) - except smtplib.SMTPServerDisconnected: + except smtplib.SMTPServerDisconnected: # pragma: no cover pass # Not connected def start(self): diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 2b135c43..f348b2c4 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -41,7 +41,12 @@ actionstop = actioncheck = -actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs} +actionban = oifs=${IFS}; + RESOLVER_ADDR="%(addr_resolver)s" + if [ "" -gt 0 ]; then echo "try to resolve $RESOLVER_ADDR"; fi + ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | tr -d '"') + IFS=,; ADDRESSES=$(echo $ADDRESSES) + IFS=${oifs} IP= FROM= SERVICE= @@ -51,26 +56,37 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(di PORT= DATE=`LC_ALL=C date --date=@