diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..86ce487f --- /dev/null +++ b/.codespellrc @@ -0,0 +1,12 @@ +[codespell] +# THANKS - names +skip = .git,*.pdf,*.svg,venv,.codespellrc,.typos.toml,THANKS,*test*.log,logs +check-hidden = true +# Ignore all acronyms etc as plenty e.g. in fail2ban/server/strptime.py +# Try to identify incomplete words which are part of a regex, hence having [] at the beginning +# Ignore all urls as something with :// in it +# Ignore all lines with codespell-ignore in them for pragma annotation +ignore-regex = (\b([A-Z][A-Z][A-Z]+|gir\.st)\b)|\[[a-zA-Z]+\][a-z]+\b|[a-z]+://\S+|.*codespell-ignore.* +# some oddly named variables, some names, etc +# wee -- comes in regex etc for weeks +ignore-words-list = assertIn,theis,timere,alls,wee,wight,ans,re-use,pre-emptive diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 543f316a..74c8eb14 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,5 @@ # These are supported funding model platforms github: [sebres] -custom: [paypal.me/sebres] +custom: [https://paypal.me/sebres] +liberapay: sebres diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 350d6ee2..b53003d5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,10 @@ Before submitting your PR, please review the following checklist: -- [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement - against certain release version, choose `0.9`, `0.10` or `0.11` branch, - for dev-edition use `master` branch - [ ] **CONSIDER adding a unit test** if your PR resolves an issue -- [ ] **LIST ISSUES** this PR resolves +- [ ] **LIST ISSUES** this PR resolves or describe the approach in detail - [ ] **MAKE SURE** this PR doesn't break existing tests -- [ ] **KEEP PR small** so it could be easily reviewed. +- [ ] **KEEP PR small** so it could be easily reviewed - [ ] **AVOID** making unnecessary stylistic changes in unrelated code - [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines - within `fail2ban/tests/files/logs/X` file + (and `# failJSON`) within `fail2ban/tests/files/logs/X` file +- [ ] **PROVIDE ChangeLog** entry describing the pull request diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000..7373affc --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,22 @@ +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Codespell + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff31db19..cde56d34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,18 +19,18 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3] + python-version: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13', '3.14.0-rc.2', pypy3.11] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -51,25 +51,35 @@ jobs: - name: Install dependencies run: | - if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi - if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then - #pip install 2to3 - sudo apt-get -y install 2to3 - fi + #if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' python -m pip install pyinotify || echo 'inotify not available' + sudo apt-get -y install sqlite3 || echo 'sqlite3 not available' #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable' python -m pip install systemd-python || echo 'systemd not available' + # readline if available as module: + python -c 'import readline' 2> /dev/null || python -m pip install readline || echo 'readline not available' + # asyncore/asynchat: + if dpkg --compare-versions "$F2B_PYV" ge 3.12; then + #sudo apt-get -y install python${F2B_PY/2/}-setuptools || echo 'setuptools not unavailable' + python -m pip install setuptools || echo "can't install setuptools" + # don't install async* modules, we need to cover bundled-in libraries: + #python -m pip install pyasynchat || echo "can't install pyasynchat"; + #python -m pip install pyasyncore || echo "can't install pyasyncore"; + fi + # aiosmtpd in test_smtp (for 3.10+, no need to test it everywhere): + if dpkg --compare-versions "$F2B_PYV" ge 3.10; then + #sudo apt-get -y install python${F2B_PY/2/}-aiosmtpd || echo 'aiosmtpd not available' + python -m pip install aiosmtpd || echo 'aiosmtpd not available' + fi - name: Before scripts run: | cd "$GITHUB_WORKSPACE" - # Manually execute 2to3 for now - if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; } # (debug) output current preferred encoding: - _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + echo 'Encodings:' $(python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))') # (debug) backend availabilities: echo 'Backends:' _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' @@ -78,14 +88,8 @@ jobs: - name: Test suite run: | - if [[ "$F2B_PY" = 2 ]]; then - python setup.py test - elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then - python bin/fail2ban-testcases --verbosity=2 - else - echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." - python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" - fi + #python setup.py test + python bin/fail2ban-testcases --verbosity=2 #- name: Test suite (debug some systemd tests only) #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..694c77d2 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,32 @@ +name: Upload Package to PyPI + +on: + workflow_dispatch: + release: + types: [created] + +jobs: + pypi-publish: + name: Publish release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/fail2ban + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip || echo "can't upgrade pip" + pip install setuptools wheel || echo "can't install/update setuptools or wheel" + - name: Build package + run: | + # python -m build ... + python setup.py sdist bdist_wheel + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 780ecfb5..5f1b8924 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ htmlcov __pycache__ .vagrant/ .idea/ +.venv/ diff --git a/.project b/.project deleted file mode 100644 index 6d59cd24..00000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - fail2ban-unstable - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 502af5be..00000000 --- a/.travis.yml +++ /dev/null @@ -1,82 +0,0 @@ -# vim ft=yaml -# travis-ci.org definition for Fail2Ban build -# https://travis-ci.org/fail2ban/fail2ban/ - -#os: linux - -language: python -dist: xenial - -matrix: - fast_finish: true - include: - - python: 2.7 - #- python: pypy - - python: 3.4 - - python: 3.5 - - python: 3.6 - - python: 3.7 - - python: 3.8 - - python: 3.9-dev - - python: pypy3.5 -before_install: - - echo "running under $TRAVIS_PYTHON_VERSION" - - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY=3; fi - - echo "Set F2B_PY=$F2B_PY" - - travis_retry sudo apt-get update -qq - # Set this so sudo executes the correct python binary - # Anything not using sudo will already have the correct environment - - export VENV_BIN="$VIRTUAL_ENV/bin" && echo "VENV_BIN set to $VENV_BIN" -install: - # Install Python packages / dependencies - # coverage - - travis_retry pip install coverage - # coveralls (note coveralls doesn't support 2.6 now): - #- if [[ $TRAVIS_PYTHON_VERSION != 2.6* ]]; then F2B_COV=1; else F2B_COV=0; fi - - F2B_COV=1 - - if [[ "$F2B_COV" = 1 ]]; then travis_retry pip install coveralls; fi - # codecov: - - travis_retry pip install codecov - # dnspython or dnspython3 - - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython || echo 'not installed'; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3 || echo 'not installed'; fi - # python systemd bindings: - - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get install -qq python-systemd || echo 'not installed'; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get install -qq python3-systemd || echo 'not installed'; fi - # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then (travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/) || echo 'not installed'; fi - # pyinotify - - travis_retry pip install pyinotify || echo 'not installed' - # Install helper tools - - sudo apt-get install shellcheck -before_script: - # Manually execute 2to3 for now - - if [[ "$F2B_PY" = 3 ]]; then ./fail2ban-2to3; fi - # (debug) output current preferred encoding: - - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' -script: - # Keep the legacy setup.py test approach of checking coverage for python2 - - if [[ "$F2B_PY" = 2 ]]; then coverage run setup.py test; fi - # Coverage doesn't pick up setup.py test with python3, so run it directly (with same verbosity as from setup) - - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi - # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) - - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' - # Test initd script - - shellcheck -s bash -e SC1090,SC1091 files/debian-initd -after_success: - - if [[ "$F2B_COV" = 1 ]]; then coveralls; fi - - codecov - -# Might be worth looking into -#notifications: -# email: true -# irc: -# channels: "irc.freenode.org#fail2ban" -# template: -# - "%{repository}@%{branch}: %{message} (%{build_url})" -# on_success: change -# on_failure: change -# skip_join: true diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 00000000..f7ef8ae1 --- /dev/null +++ b/.typos.toml @@ -0,0 +1,35 @@ +[files] +extend-exclude = [ + ".git/", + ".codespellrc", + "fail2ban/tests/files/logs/", +] +ignore-hidden = false + +[default] +extend-ignore-re = [ + "Christoph Theis", + "\\[[0-9a-f]{7,8}\\]", + "hash_[0-9a-f]{38}", + "\t[0-9.-]+[ A-Z]+", + "Erreur d'authentification", + "WebEMailExtrac", + "ssh2: RSA 14:ba:xx", + "\\[Cc\\]lient", + "\\[Gg\\]ot", + "\\[nN\\]ot", + "\\[Uu\\]nknown", + "\\[uU\\]ser", + "\\[Uu\\]\\(\\?:ser", +] + +[default.extend-words] +"alls" = "alls" +"helo" = "helo" + +[default.extend-identifiers] +"failManager2nd" = "failManager2nd" +"log2nd" = "log2nd" +"routeros" = "routeros" +"REFERERS" = "REFERERS" +"tre_search" = "tre_search" diff --git a/ChangeLog b/ChangeLog index cb2959a5..b1ce9327 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,7 +7,204 @@ Fail2Ban: Changelog =================== -ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition +ver. 1.1.1-dev-1 (20??/??/??) - development nightly edition +----------- + +### Compatibility +* `action.d/iptables.conf` rewritten due to support of multiple chains (gh-3909), therefore user-level derivations + (action including iptables-based action) may become incompatible, e. g. some tags if used need to be replaced, + e. g. `` with `$chain` or `<_ipt_for_proto-iter>` with `<_ipt-iter>`; +* `filter.d/exim.conf` - several rules of mode `normal` moved to new mode `more`, because of too risky handling (see [gh-3940](https://github.com/fail2ban/fail2ban/pull/3940)), + to use it as before set `mode = more` for exim jail, but be aware of the consequences. + +### Fixes +* fixes `systemd` bug with missing journal descriptor after rotation by reopening of journal if it is recognized as not alive (gh-3929) +* improve threaded clean-up of all filters, new thread functions `afterStop` (to force clean-up after stop) and `done`, invoking `afterStop` once +* ensure journal-reader is always closed (additional prevention against leaks and "too many open files"), thereby avoid sporadic segfault + in systemd module (see https://github.com/systemd/python-systemd/issues/143) +* fixes `systemd` causing "too many open files" error for a lot of journal files and large amount of systemd jails + (see new parameter `rotated` below, gh-3391); +* passing of arguments from jails to action or filter will affect conditional section too (gh-4069), + e. g. setting `blocktype="DROP"` via jail for action would now apply for IPv4 and IPv6 chains, + to submit different `blocktype` for IPv4 and IPv6 from jail, one can pass them like in this example: + `banaction = iptables-ipset[blocktype="...", blocktype?family=inet6="..."]` +* `jail.conf`: + - default banactions need to be specified in `paths-*.conf` (maintainer level) now + - since stock fail2ban includes `paths-debian.conf` by default, banactions are `nftables` + (can be overwritten in `jail.local` by user) +* `paths-common.conf`: + - changed default `mysql_log` path (default `logpath` of `mysqld-auth` jail without maintainer overrides, gh-3932) +* `paths-debian.conf`: + - default banactions are `nftables` + - sshd backend switched to `systemd` (gh-3292) + - postfix backend switched to `systemd` (gh-3527) +* `action.d/firewallcmd-ipset.conf`: + - rename `ipsettype` to `ipsetbackend` (gh-2620), parameter `ipsettype` will be used now to the real set type (gh-3760) +* `action.d/xarf-login-attack.conf` - ignore errors or warnings in output of `dig` provided as comment (gh-4068) +* `filter.d/apache-badbots.conf`, `filter.d/apache-fakegooglebot.conf`: + - regexs rewritten more strict (removed catch-alls, etc); + - regexs fixed to match lines with vhost in accesslog (gh-1594) +* `filter.d/apache-noscript.conf` - consider new log-format with "AH02811: stderr from /..." (gh-3900) +* `filter.d/apache-overflows.conf` - consider AH10244: invalid URI path (gh-3778, gh-3900) +* `filter.d/asterisk.conf` - fixed RE for "no matching endpoint" with retry info (like `after X tries in Y ms`) at end, + loosening of end anchor (ignore any simple text tokens at end if no single quote found), gh-4037 +* `filter.d/exim.conf`: + - several rules of mode `normal` moved to new mode `more`, because of too risky handling (gh-3940), + thereby mode `aggressive` is not affected, because it fully includes mode `more` now; + - mode `aggressive` extended to catch dropped by ACL failures, e.g. "ACL: Country is banned" +* `filter.d/freeswitch.conf` - bypass some new info in prefix before [WARNING] (changed default `_pref_line`), + FreeSWITCH log line prefix has changed in newer versions (gh-3143) +* `filter.d/lighttpd-auth.conf` - fixed regex (if failures generated by systemd-journal), bypass several prefixes now (gh-3955) +* `filter.d/postfix.conf`: + - consider CONNECT and other rejected commands as a valid `_pref` (gh-3800) + - default `_daemon` in prefix-line is loosened - can match everything starting with word postfix, like `postfix-example.com/smtpd` (gh-3297) + - add optional `NOQUEUE:` prefix to ddos regex (gh-4072) + - internal parameter `_pref` is renamed to `_cmd`, `_pref` matches now optional prefix like `NOQUEUE: ` etc + - modes `ddos` and `aggressive` extended to match `rate limit exceeded` for connection or message delivery request rates (gh-3265, gh-4073) +* `filter.d/dropbear.conf`: + - recognizes extra pid/timestamp if logged into stdout/journal, added `journalmatch` (gh-3597) + - failregex extended to match different format of "Exit before auth" message (gh-3791) +* `filter.d/recidive.conf` - restore possibility to set jail name in the filter, _jailname is positive now (gh-3769) +* `filter.d/roundcube-auth.conf` - improved RE better matching log format of roundcube version 1.4+ (gh-3816) +* `filter.d/sendmail-reject.conf`: (gh-4020) + - support `` for BSD-style logfiles + - add match for `User unknown` to default + - the relay field may not always have a hostname before the ip address + - mode `aggressive` enables match for `lost input channel` and `Cannot resolve PTR record` +* `filter.d/sshd.conf`: + - adapted to conform possible new daemon name sshd-session, since OpenSSH 9.8 + several log messages will be tagged with as originating from a process named "sshd-session" rather than "sshd" (gh-3782) + - `ddos` and `aggressive` modes: regex extended for timeout before authentication (optional connection from part, gh-3907) +* `filter.d/vsftpd.conf` - fixed regex (if failures generated by systemd-journal, gh-3954) +* `filter.d/froxlor-auth.conf` - updated the regex to the new logging situation for froxlor and changed logpath in jail.conf (gh-4075). + +### New Features and Enhancements +* backend `systemd` extended with new parameter `rotated` (default `false`, as prevention against "too many open files"), + that allows to monitor only actual journals and ignore now a lot of rotated files by default; so can drastically reduce + amount of used file descriptors, normally to 1 or 2 descriptors per jail (gh-3391) +* new jail option `skip_if_nologs` to ignore jail if no `logpath` matches found, fail2ban continue to start with warnings/errors, + thus other jails become running (gh-2756) +* implements automatic switch `backend = auto` to backend `systemd`, when the following is true (RFE gh-3768): + - no files matching `logpath` found for this jail; + - no `systemd_if_nologs = false` is specified for the jail (`true` by default); + - option `journalmatch` is set for the jail or its filter (otherwise it'd be too heavy to allow all auto-jails, + even if they have never been foreseen for journal monitoring); + (option `skip_if_nologs` will be ignored if we could switch backend to `systemd`) +* configuration `ignoreip` and fail2ban-client commands `addignoreip`/`delignoreip` extended with `file:...` syntax + to ignore IPs from file-ip-set (containing IP, subnet, dns/fqdn or raw strings); the file would be read lazy on demand, + by first ban (and automatically reloaded by update after small latency to avoid expensive stats check on every compare); + the entries inside the file can be separated by comma, space or new line with optional comments (text following chars + `#` or `;` after space or newline would be ignored up to next newline) +* `action.d/*-ipset.conf`: + - parameter `ipsettype` to set type of ipset, e. g. hash:ip, hash:net, etc (gh-3760) +* `action.d/iptables.conf` - action and few derivatives of it extended to handle multiple chains, + e. g. would also accept `chain = INPUT,FORWARD` (gh-3909) +* `action.d/nftables.conf` (gh-3291): + - new parameter `addr_options` for addr-set (default `flags interval\;`, allows to store CIDR or address ranges); + can be set to empty value to create simple addresses set (restore previous behavior). +* `action.d/firewallcmd-rich-*.conf` - fixed incorrect quoting, disabling port variable expansion + by substitution of rich rule (gh-3815) +* `filter.d/dovecot.conf`: + - add support for latest Dovecot 2.4 release (gh-4016) + - mode `aggressive` covered new variant for `no auth attempts in X secs` with `Login aborted` and `(no_auth_attempts)` + - mode `aggressive` extended to match `disconnected during TLS handshake` with `no application protocol` and `no shared cipher` +* `filter.d/nginx-http-auth.conf`: + - extended with `prefregex` to capture content of error only (bypass common prefix and suffix, like server, request, host, referrer); + - extended to match PAM authentication failures (gh-4071) +* `filter.d/nginx-limit-req.conf` - extended to ban hosts failed by limit connection in ngx_http_limit_conn_module (gh-3674, gh-4047) +* `filter.d/proxmox.conf` - add support to Proxmox Web GUI (gh-2966) +* `filter.d/openvpn.conf` - new filter and jail for openvpn recognizing failed TLS handshakes (gh-2702) +* `filter.d/sendmail-reject.conf` - also recognize "Domain of sender address ... does not resolve" (gh-4035) +* `filter.d/vaultwarden.conf` - new filter and jail for Vaultwarden (gh-3979) +* `filter.d/xrdp.conf` - new filter for XRDP, an open source RDP server (gh-3254) +* `fail2ban-regex` extended with new option `-i` or `--invert` to output not-matched lines by `-o` or `--out` (gh-4001) + + +ver. 1.1.0 (2024/04/25) - object-found--norad-59479-cospar-2024-069a--altitude-36267km +----------- + +### Compatibility +* the minimum supported python version is now 3.5, if you have previous python version + you can use the 0.11 or 1.0 version of fail2ban or upgrade python (or even build it from source). + +### Fixes +* circumvent SEGFAULT in a python's socket module by getaddrinfo with disabled IPv6 (gh-3438) +* avoid sporadic error in pyinotify backend if pending file deleted in other thread, e. g. by flushing logs (gh-3635) +* `action.d/cloudflare-token.conf` - fixes gh-3479, url-encode args by unban +* `action.d/*ipset*`: make `maxelem` ipset option configurable through banaction arguments (gh-3564) +* `filter.d/apache-common.conf` - accepts remote besides client (gh-3622) +* `filter.d/mysqld-auth.conf` - matches also if no suffix in message (mariadb 10.3 log format, gh-3603) +* `filter.d/nginx-*.conf` - nginx error-log filters extended with support of journal format (gh-3646) +* `filter.d/postfix.conf`: + - "rejected" rule extended to match "Access denied" too (gh-3474) + - avoid double counting ('lost connection after AUTH' together with message 'disconnect ...', gh-3505) + - add Sender address rejected: Malformed DNS server reply (gh-3590) + - add to postfix syslog daemon format (gh-3690) + - change journalmatch postfix, allow sub-units with postfix@-.service (gh-3692) +* `filter.d/recidive.conf`: support for systemd-journal, conditional RE depending on logtype (for file or journal, gh-3693) +* `filter.d/slapd.conf` - filter rewritten for single-line processing, matches errored result without `text=...` (gh-3604) + +### New Features and Enhancements +* supports python 3.12 and 3.13 (gh-3487) +* bundling async modules removed in python 3.12+ (fallback to local libraries pyasyncore/pyasynchat if import would miss them, gh-3487) +* `fail2ban-client` extended (gh-2975): + - `fail2ban-client status --all [flavor]` - returns status of fail2ban and all jails in usual form + - `fail2ban-client stats` - returns statistic in form of table (jail, backend, found and banned counts) + - `fail2ban-client statistic` or `fail2ban-client statistics` - same as `fail2ban-client stats` (aliases for stats) + - `fail2ban-client status --all stats` - (undocumented, flavor "stats") returns statistic of all jails in form of python dict +* `fail2ban-regex` extended to load settings from jail (by simple name it'd prefer jail to the filter now, gh-2655); + to load the settings from filter one could use: +```diff +- fail2ban-regex ... sshd ; # jail ++ fail2ban-regex ... sshd.conf ; # filter +# or: ++ fail2ban-regex ... filter.d/sshd ; # filter +``` +* better auto-detection for IPv6 support (`allowipv6 = auto` by default), trying to check sysctl net.ipv6.conf.all.disable_ipv6 + (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces + if available for platform and uses DNS to find local IPv6 as a fallback only +* improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132) +* `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860) +* `action.d/pf.conf` - pf action extended with support of `protocol=all` (gh-3503) +* `action.d/smtp.py` - added optional support for TLS connections via the `ssl` arg. +* `filter.d/dante.conf` - new filter for Dante SOCKS server (gh-2112) +* `filter.d/exim.conf`, `filter.d/exim-spam.conf`: + - messages are prefiltered by `prefregex` now + - filter can bypass additional timestamp or pid that may be logged via systemd-journal or syslog-ng (gh-3060) + - rewrite host line regex for all varied exim's log_selector states (gh-3263, gh-3701, gh-3702) + - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502) +* `filter.d/named-refused.conf` - denied allows any reason in parenthesis as suffix (gh-3697) +* `filter.d/nginx-forbidden.conf` - new filter to ban forbidden locations, e. g. using `deny` directive (gh-2226) +* `filter.d/routeros-auth.conf` - new filter detecting failed login attempts in the log produced by MikroTik RouterOS +* `filter.d/sshd.conf`: + - avoid double counting for "maximum authentication attempts exceeded" (gh-3502) + - message "Disconnecting ... Too many authentication failures" is not a failure anymore + - mode `ddos`/`aggressive` extended to match new messages caused by port scanner, wrong payload on ssh port (gh-3486): + * message authentication code incorrect [preauth] + * connection corrupted [preauth] + * timeout before authentication + + +ver. 1.0.2 (2022/11/09) - finally-war-game-test-tape-not-a-nuclear-alarm +----------- + +### Fixes +* backend `systemd`: code review and several fixes: + - wait only if it is necessary, e. g. in operational mode and if no more entries retrieved (end of journal); + - ensure we give enough time after possible rotation, vacuuming or adding/removing journal files, + and move cursor back and forth to avoid entering dead space +* `filter.d/named-refused.conf`: + - support BIND named log categories, gh-3388 + - allow `info:` as possible error prefix too ("query (cache) denied" may occur as info) +* `filter.d/dovecot.conf`: + - fixes regression introduced in gh-3210: resolve extremely long search by repeated apply of non-greedy RE-part + with following branches (it may be extremely slow up to infinite search depending on message), gh-3370 + - fixes regression and matches new format in aggressive mode too (amend to gh-3210) + +### New Features and Enhancements + + +ver. 1.0.1 (2022/09/27) - energy-equals-mass-times-the-speed-of-light-squared ----------- ### Compatibility @@ -15,6 +212,10 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition you can use the 0.11 version of fail2ban or upgrade python (or even build it from source). * potential incompatibility by parsing of options of `backend`, `filter` and `action` parameters (if they are partially incorrect), because fail2ban could throw an error now (doesn't silently bypass it anymore). +* due to fix for CVE-2021-32749 (GHSA-m985-3f3v-cwmm) the mailing action using mailutils may require extra configuration, + if it is not compatible or doesn't support `-E 'set escape'` (e. g. with `mailcmd` parameter), see gh-3059 +* automatic invocation of 2to3 is removed in setup now (gh-3098), there is also no option `--disable-2to3` anymore, + `./fail2ban-2to3` should be called outside before setup * to v.0.11: - due to change of `actioncheck` behavior (gh-488), some actions can be incompatible as regards the invariant check, if `actionban` or `actionunban` would not throw an error (exit code @@ -24,28 +225,144 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition and the values are different (gh-3217) ### Fixes +* theoretical RCE vulnerability in mailing action using mailutils (mail-whois), CVE-2021-32749, GHSA-m985-3f3v-cwmm * readline fixed to consider interim new-line character as part of code point in multi-byte logs (e. g. unicode encoding like utf-16be, utf-16le); +* [stability] solves race condition with uncontrolled growth of failure list (jail with too many matches, + that did not cause ban), behavior changed to ban ASAP, gh-2945 +* fixes search for the best datepattern - e. g. if line is too short, boundaries check for previously known + imprecise pattern may fail on incomplete lines (logging break-off, no flush, etc), gh-3020 +* [stability, performance] backend `systemd`: + - fixes error "local variable 'line' referenced before assignment", introduced in 55d7d9e2, gh-3097 + - don't update database too often (every 10 ticks or ~ 10 seconds in production) + - fixes wrong time point of "in operation" mode, gh-2882 + - better avoidance of landing in dead space by seeks over journals (improved seek to time) + - fixes missing space in message (tag ``) between timestamp and host if the message read from systemd journal, gh-3293 +* [stability] backend `pyinotify`: fixes sporadic runtime error "dictionary changed size during iteration" +* several backends optimizations (in file and journal filters): + - don't need to wait if we still had log-entries from last iteration (which got interrupted for servicing) + - rewritten update log/journal position, it is more stable and faster now (fewer DB access and surely up-to-date at end) +* `paths-debian.conf`: + - add debian path to roundcube error logs +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-`;" + reverted the incompatibility gh-3047 introduced in a038fd5, gh-2821, because this depends now on firewalld backend + (e. g. `-` vs. `:` related to `iptables` vs. `nftables`) +* `action.d/nginx-block-map.conf`: reload nginx only if it is running (also avoid error in nginx-errorlog, gh-2949) * `action.d/ufw.conf`: - fixed handling on IPv6 (using prepend, gh-2331, gh-3018) - application names containing spaces can be used now (gh-656, gh-1532, gh-3018) -* `filter.d/drupal-auth.conf` more strict regex, extended to match "Login attempt failed from" (gh-2742) +* `filter.d/apache-fakegooglebot.conf`: + - better, more precise regex and datepattern (closes possible weakness like gh-3013) + - `filter.d/ignorecommands/apache-fakegooglebot` - added timeout parameter (default 55 seconds), avoid fail with timeout + (default 1 minute) by reverse lookup on some slow DNS services (googlebots must be resolved fast), gh-2951 +* `filter.d/apache-overflows.conf` - extended to match AH00126 error (Invalid URI ...), gh-2908 +* `filter.d/asterisk.conf` - add transport to asterisk RE: call rejection messages can have the transport prefixed to the IP address, gh-2913 +* `filter.d/courier-auth.conf`: + - consider optional port after IP, gh-3211 + - regex is rewritten without catch-all's and right anchor, so it is more stable against further modifications now +* `filter.d/dovecot.conf`: + - adjusted for updated dovecot log format with `read(size=...)` in message (gh-3210) + - parse everything in parenthesis by auth-worker info, e. g. can match (pid=...,uid=...) too (amend to gh-2553) + - extended to match prefix like `conn unix:auth-worker (uid=143): auth-worker<13247>:` + (authenticate from external service like exim), gh-2553 + - fixed "Authentication failure" regex, matches "Password mismatch" in title case (gh-2880) +* `filter.d/drupal-auth.conf` - more strict regex, extended to match "Login attempt failed from" (gh-2742) +* `filter.d/exim-common.conf` - pid-prefix extended to match `mx1 exim[...]:` (gh-2553) +* `filter.d/lighttpd-auth.conf` - adjusted to the current source code + avoiding catch-all's, etc (gh-3116) +* `filter.d/named-refused.conf`: + - added support for alternate names (suffix), FreeIPA renames the BIND9 named daemon to named-pkcs11, gh-2636 + - fixes prefix for messages from systemd journal (no mandatory space ahead, because don't have timestamp), gh-2899 +* `filter.d/nginx-*.conf` - added journalmatch to nginx filters, gh-2935 +* `filter.d/nsd.conf` - support for current log format, gh-2965 +* `filter.d/postfix.conf`: fixes and new vectors, review and combining several regex to single RE: + - mode `ddos` (and `aggressive`) extended: + * to consider abusive handling of clients hitting command limit, gh-3040 + * to handle postscreen's PREGREET and HANGUP messages, gh-2898 + - matches rejects with "undeliverable address" (sender/recipient verification) additionally to "Unknown user", gh-3039 + both are configurable now via extended parameter and can be disabled using `exre-user=` supplied in filter parameters + - reject: BDAT/DATA from, gh-2927 + - (since regex is more precise now) token selector changed to `[A-Z]{4}`, e. g. no matter what a command is supplied now + (RCPT, EHLO, VRFY, DATA, BDAT or something else) + - matches "Command rejected" and "Data command rejected" now + - matches RCPT from unknown, 504 5.5.2, need fully-qualified hostname, gh-2995 + - matches 550 5.7.25 Client host rejected, gh-2996 +* `filter.d/sendmail-auth.conf`: + - detect several "authentication failure" messages, sendmail 8.16.1, gh-2757 + - detect user not found, gh-3030 + - detect failures without user part, gh-3324 +* `filter.d/sendmail-reject.conf`: + - fix reverse DNS for ... (gh-3012) + - fixed regex to consider "Connection rate limit exceeded" with different combination of arguments +* `filter.d/sshd.conf`: + - mode `ddos` extended - recognizes messages "kex_exchange_identification: Connection closed / reset by pear", gh-3086 + (fixed possible regression of f77398c) + - mode `ddos` extended - recognizes new message "banner exchange: invalid format" generated by port scanner + (https payload on ssh port), gh-3169 +* `filter.d/zoneminder.conf` - support new log format (ERR instead of WAR), add detection of non-existent user login attempts, gh-2984 +* amend to gh-980 fixing several actions (correctly supporting new enhancements now) +* fixed typo by `--dump-pretty` option which did never work (only `--dp` was working) +* fixes start of fail2ban-client in docker: speedup daemonization process by huge open files limit, gh-3334 +* provides details of failed regex compilation in the error message we throw in Regex-constructor + (it's good to know what exactly is wrong) +* fixed failed update of database didn't signal with an error, gh-3352: + - client and server exit with error code by failure during start process (in foreground mode) + - added fallback to repair if database cannot be upgraded ### New Features and Enhancements +* python 3.10 and 3.11 compatibility (and GHA-CI support) * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) * better recognition of log rotation, better performance by reopen: avoid unnecessary seek to begin of file (and hash calculation) * file filter reads only complete lines (ended with new-line) now, so waits for end of line (for its completion) +* datedetector: + - token `%Z` must recognize zone abbreviation `Z` (GMT/UTC) also (similar to `%z`) + - token `%Z` recognizes all known zone abbreviation besides Z, GMT, UTC correctly, if it is matching + (`%z` remains unchanged for backwards-compatibility, see comment in code) + - date patterns `%ExY` and `%Exy` accept every year from 19xx up to current century (+3 years) in `fail2ban-regex` + - better grouping algorithm for resulting century RE for `%ExY` and `%Exy` * actions differentiate tags `` and `` (``), if IP-address deviates from ID then the value of `` is not equal `` anymore (gh-3217) +* action info extended with new members for jail info (usable as tags in command actions), gh-10: + - ``, `` - current and total found failures + - ``, `` - current and total bans +* `filter.d/monitorix.conf` - added new filter and jail for Monitorix, gh-2679 +* `filter.d/mssql-auth.conf` - new filter and jail for Microsoft SQL Server, gh-2642 +* `filter.d/nginx-bad-request.conf` - added filter to find bad requests (400), gh-2750 +* `filter.d/nginx-http-auth.conf` - extended with parameter mode, so additionally to `auth` (or `normal`) + mode `fallback` (or combined as `aggressive`) can find SSL errors while SSL handshaking, gh-2881 +* `filter.d/scanlogd.conf` - new filter and jail, add support for filtering out detected port scans via scanlogd, gh-2950 +* `action.d/apprise.conf` - added Apprise support (50+ Notifications), gh-2565 +* `action.d/badips.*` - removed actions, badips.com is no longer active, gh-2889 +* `action.d/cloudflare.conf` - better IPv6 capability, gh-2891 +* `action.d/cloudflare-token.conf` - added support for Cloudflare Token APIs. This method is more restrictive and therefore safter than using API Keys. +* `action.d/ipthreat.conf` - new action for IPThreat integration, gh-3349 * `action.d/ufw.conf` (gh-3018): - new option `add` (default `prepend`), can be supplied as `insert 1` for ufw versions before v.0.36 (gh-2331, gh-3018) - new options `kill-mode` and `kill` to drop established connections of intruder (see action for details, gh-3018) -* `filter.d/nginx-http-auth.conf` - extended with parameter mode, so additionally to `auth` (or `normal`) - mode `fallback` (or combined as `aggressive`) can find SSL errors while SSL handshaking, gh-2881 -* `filter.d/xrdp.conf` - new filter for XRDP, an open source RDP server +* `iptables` and `iptables-ipset` actions extended to support multiple protocols with single action + for multiport or oneport type (back-ported from nftables action); +* `iptables` actions are more breakdown-safe: start wouldn't fail if chain or rule already exists + (e. g. created by previous instance and doesn't get purged properly); ultimately closes gh-980 +* `ipset` actions are more breakdown-safe: start wouldn't fail if set with this name already exists + (e. g. created by previous instance and don't deleted properly) +* replace internals of several `iptables` and `iptables-ipset` actions using internals of iptables include: + - better check mechanism (using `-C`, option `--check` is available long time); + - additionally iptables-ipset is a common action for `iptables-ipset-proto6-*` now (which become obsolete now); + - many features of different iptables actions are combinable as single chain/rule (can be supplied to action as parameters); + - iptables is a replacement for iptables-common now, several actions using this as include now become obsolete; +* new logtarget SYSTEMD-JOURNAL, gh-1403 +* fail2ban.conf: new fail2ban configuration option `allowipv6` (default `auto`), can be used to allow or disallow IPv6 + interface in fail2ban immediately by start (e. g. if fail2ban starts before network interfaces), gh-2804 +* invalidate IP/DNS caches by reload, so inter alia would allow to recognize IPv6IsAllowed immediately, previously + retarded up to cache max-time (5m), gh-2804 +* OpenRC (Gentoo, mainly) service script improvements, gh-2182 +* suppress unneeded info "Jail is not a JournalFilter instance" (moved to debug level), gh-3186 +* implements new interpolation variable `%(fail2ban_confpath)s` (automatically substituted from config-reader path, + default `/etc/fail2ban` or `/usr/local/etc/fail2ban` depending on distribution); `ignorecommands_dir` is unneeded anymore, + thus removed from `paths-common.conf`, fixes gh-3005 +* `fail2ban-regex`: accepts filter parameters containing new-line ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools @@ -252,7 +569,7 @@ filter = flt[logtype=short] * `filter.d/znc-adminlog.conf`: new filter for ZNC (IRC bouncer); requires the adminlog module to be loaded ### Enhancements -* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to contol +* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to control how many matches per ticket fail2ban can hold in memory and store in database (gh-2402, gh-2118); * fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to @@ -362,7 +679,7 @@ ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right - fixed root login refused regex (optional port before preauth, gh-2080); - avoid banning of legitimate users when pam_unix used in combination with other password method, so bypass pam_unix failures if accepted available for this user gh-2070; - - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediatelly); + - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediately); - mode `ddos` (and `aggressive`) extended to catch `Connection closed by ... [preauth]`, so in DDOS mode it counts failure on closing connection within preauth-stage (gh-2085); * `action.d/abuseipdb.conf`: fixed curl cypher errors and comment quote-issue (gh-2044, gh-2101); @@ -692,7 +1009,7 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc sane environment in error case of `actioncheck`. * Reporting via abuseipdb.com: - Bans can now be reported to abuseipdb - - Catagories must be set in the config + - Categories must be set in the config - Relevant log lines included in report ### Enhancements @@ -829,7 +1146,7 @@ releases. - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs. * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882) * filter.d/asterisk.conf - - fixed failregex AMI Asterisk authentification failed (see gh-1302) + - fixed failregex AMI Asterisk authentication failed (see gh-1302) - removed invalid (vulnerable) regex blocking IPs using forign data (from header "from") thus not the IP-address that really originates the request (see gh-1927) - fixed failregex for the SQL-injection attempts with single-quotes in connect-string (see gh-2011) @@ -1129,7 +1446,7 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends * `filter.d/roundcube-auth.conf` - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log -* `action.d/sendmail*.conf` - use LC_ALL (superseeding LC_TIME) to override +* `action.d/sendmail*.conf` - use LC_ALL (superseding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at communication end (gh-1099) @@ -1299,7 +1616,7 @@ ver. 0.9.1 (2014/10/29) - better, faster, stronger * Ignored IPs are no longer banned when being restored from persistent database * Manually unbanned IPs are now removed from persistent database, such they - wont be banned again when Fail2Ban is restarted + won't be banned again when Fail2Ban is restarted * Pass "bantime" parameter to the actions in default jail's action definition(s) * `filters.d/sieve.conf` - fixed typo in _daemon. Thanks Jisoo Park @@ -1590,7 +1907,7 @@ those filters were used. all platforms to ensure permissions are the same before and after a ban. Closes gh-266. hostsdeny supports daemon_list now too. * `action.d/bsd-ipfw` - action option unused. Change blocktype to port unreach - instead of deny for consistancy. + instead of deny for consistency. * `filter.d/dovecot` - added to support different dovecot failure "..disallowed plaintext auth". Closes Debian bug #709324 * `filter.d/roundcube-auth` - timezone offset can be positive or negative @@ -1780,7 +2097,7 @@ fail2ban-users mailing list and IRC. ### New Features - Yaroslav Halchenko * [9ba27353] Add support for `jail.d/{confilefile}` and `fail2ban.d/{configfile}` - to provide additional flexibility to system adminstrators. Thanks to + to provide additional flexibility to system administrators. Thanks to beilber for the idea. Closes gh-114. * [3ce53e87] Add exim filter. - Erwan Ben Souiden @@ -1931,7 +2248,7 @@ ver. 0.8.7 (2012/07/31) - stable * [47c03a2] files/nagios - spelling/grammar fixes * [b083038] updated Free Software Foundation's address * [9092a63] changed TLDs to invalid domains, in accordance with RFC 2606 - * [642d9af,3282f86] reformated printing of jail's name to be consistent + * [642d9af,3282f86] reformatted printing of jail's name to be consistent with init's info messages * [3282f86] uniform use of capitalized Jail in the messages - Leonardo Chiquitto @@ -2276,7 +2593,7 @@ ver. 0.6.1 (2006/03/16) - stable - Fixed crash when time format does not match data - Propagated patch from Debian to fix fail2ban search path addition to the path search list: now it is added first. Thanks to Nick Craig-Wood -- Added SMTP authentification for mail notification. Thanks to Markus Hoffmann +- Added SMTP authentication for mail notification. Thanks to Markus Hoffmann - Removed debug mode as it is confusing for people - Added parsing of timestamp in TAI64N format (#1275325). Thanks to Mark Edgington @@ -2309,7 +2626,7 @@ ver. 0.5.5 (2005/10/26) - beta further adjusted by upstream author). * Added -f command line parameter for [findtime]. * Added a cleanup of firewall rules on emergency shutdown when unknown - exception is catched. + exception is caught. * Fail2ban should not crash now if a wrong file name is specified in config. * reordered code a bit so that log targets are setup right after background and then only loglevel (verbose, debug) is processed, so the warning could @@ -2398,7 +2715,7 @@ ver. 0.3.1 (2005/03/31) - beta ver. 0.3.0 (2005/02/24) - beta ---------- -- Re-writting of parts of the code in order to handle several log files with +- Re-writing of parts of the code in order to handle several log files with different rules - Removed `sshd.py` because it is no more needed - Fixed a bug when exiting with IP in the ban list diff --git a/FILTERS b/FILTERS index 2ed6281d..5304399b 100644 --- a/FILTERS +++ b/FILTERS @@ -129,7 +129,7 @@ Date/Time --------- At the moment, Fail2Ban depends on log lines to have time stamps. That is why -before starting to develop failregex, check if your log line format known to +before starting to develop failregex, check if your log line format is known to Fail2Ban. Copy the time component from the log line and append an IP address to test with following command:: @@ -232,16 +232,24 @@ the <> at the start so regex should be similar to '^<> error is evil$' us The following general rules apply to regular expressions: -* ensure regexes start with a ^ and are as restrictive as possible. E.g. do not - use .* if \d+ is sufficient; +* ensure regexes are anchored (e. g. start with a ^) and are as restrictive + as possible. E.g. do not use catch-alls .+ or .* if \d+ or [^"]* is sufficient. + Basically avoid the catch-alls where it is possible, especially non-greedy + catch-alls on RE with many branches or ambiguous matches; * use functionality of Python regexes defined in the standard Python re library - http://docs.python.org/2/library/re.html; -* make regular expressions readable (as much as possible). E.g. - (?:...) represents a non-capturing regex but (...) is more readable, thus - preferred. + https://docs.python.org/library/re.html; +* try to write regular expressions as efficient as possible. E.g. do not write + several REs for almost the same messages, just with A or B or C, if they can + be matched by single RE using | operator like ...(?:A|B|C)... and order them + by their frequency, so A before B and C, if A is more frequent or will match + faster; +* make regular expressions readable (as much as possible), but only if it is + justified. E.g. (?:...) represents a non-capturing regex and (...) is more + readable, but capturing groups make the RE a bit slower, thus (?:...) may be + more preferable. If you have only a basic knowledge of regular repressions we advise to read -http://docs.python.org/2/library/re.html first. It doesn't take long and would +https://docs.python.org/library/re.html first. It doesn't take long and would remind you e.g. which characters you need to escape and which you don't. Developing/testing a regex @@ -265,10 +273,6 @@ Take note of -l heavydebug / -l debug and -v as they might be very useful. parts are constrained and different formats depending on configuration or less common usages. -.. TIP:: - For looking through source code - http://sourcecodebrowser.com/ . It has - call graphs and can browse different versions. - .. TIP:: Some applications log spaces at the end. If you are not sure add \s*$ as the end part of the regex. @@ -309,6 +313,8 @@ So more specifically in the [filter] section in jail.conf: Submit github pull request (See "Pull Requests" above) for github.com/fail2ban/fail2ban containing your great work. +You may also consider https://github.com/fail2ban/fail2ban/wiki/Best-practice + Filter Security =============== diff --git a/MANIFEST b/MANIFEST index bf46406c..972a2b48 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,12 +3,13 @@ bin/fail2ban-regex bin/fail2ban-server bin/fail2ban-testcases ChangeLog -config/action.d/apprise.conf config/action.d/abuseipdb.conf config/action.d/apf.conf +config/action.d/apprise.conf config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf +config/action.d/cloudflare-token.conf config/action.d/complain.conf config/action.d/dshield.conf config/action.d/dummy.conf @@ -25,6 +26,7 @@ config/action.d/ipfilter.conf config/action.d/ipfw.conf config/action.d/iptables-allports.conf config/action.d/iptables.conf +config/action.d/iptables-ipset.conf config/action.d/iptables-ipset-proto4.conf config/action.d/iptables-ipset-proto6-allports.conf config/action.d/iptables-ipset-proto6.conf @@ -32,11 +34,13 @@ config/action.d/iptables-multiport.conf config/action.d/iptables-multiport-log.conf config/action.d/iptables-new.conf config/action.d/iptables-xt_recent-echo.conf +config/action.d/ipthreat.conf config/action.d/mail-buffered.conf config/action.d/mail.conf config/action.d/mail-whois-common.conf config/action.d/mail-whois.conf config/action.d/mail-whois-lines.conf +config/action.d/mikrotik.conf config/action.d/mynetwatchman.conf config/action.d/netscaler.conf config/action.d/nftables-allports.conf @@ -87,6 +91,7 @@ config/filter.d/counter-strike.conf config/filter.d/courier-auth.conf config/filter.d/courier-smtp.conf config/filter.d/cyrus-imap.conf +config/filter.d/dante.conf config/filter.d/directadmin.conf config/filter.d/domino-smtp.conf config/filter.d/dovecot.conf @@ -110,11 +115,16 @@ config/filter.d/kerio.conf config/filter.d/lighttpd-auth.conf config/filter.d/mongodb-auth.conf config/filter.d/monit.conf +config/filter.d/monitorix.conf +config/filter.d/mssql-auth.conf config/filter.d/murmur.conf config/filter.d/mysqld-auth.conf config/filter.d/nagios.conf config/filter.d/named-refused.conf +config/filter.d/nginx-bad-request.conf config/filter.d/nginx-botsearch.conf +config/filter.d/nginx-error-common.conf +config/filter.d/nginx-forbidden.conf config/filter.d/nginx-http-auth.conf config/filter.d/nginx-limit-req.conf config/filter.d/nsd.conf @@ -132,6 +142,8 @@ config/filter.d/pure-ftpd.conf config/filter.d/qmail.conf config/filter.d/recidive.conf config/filter.d/roundcube-auth.conf +config/filter.d/routeros-auth.conf +config/filter.d/scanlogd.conf config/filter.d/screensharingd.conf config/filter.d/selinux-common.conf config/filter.d/selinux-ssh.conf @@ -168,7 +180,6 @@ CONTRIBUTING.md COPYING .coveragerc DEVELOP -fail2ban-2to3 fail2ban/client/actionreader.py fail2ban/client/beautifier.py fail2ban/client/configparserinc.py @@ -184,6 +195,8 @@ fail2ban/client/filterreader.py fail2ban/client/__init__.py fail2ban/client/jailreader.py fail2ban/client/jailsreader.py +fail2ban/compat/asynchat.py +fail2ban/compat/asyncore.py fail2ban/exceptions.py fail2ban/helpers.py fail2ban/__init__.py @@ -197,7 +210,6 @@ fail2ban/server/datedetector.py fail2ban/server/datetemplate.py fail2ban/server/failmanager.py fail2ban/server/failregex.py -fail2ban/server/filtergamin.py fail2ban/server/filterpoll.py fail2ban/server/filter.py fail2ban/server/filterpyinotify.py @@ -265,7 +277,7 @@ fail2ban/tests/files/config/apache-auth/noentry/.htaccess fail2ban/tests/files/config/apache-auth/README fail2ban/tests/files/database_v1.db fail2ban/tests/files/database_v2.db -fail2ban/tests/files/filter.d/substition.conf +fail2ban/tests/files/filter.d/substitution.conf fail2ban/tests/files/filter.d/testcase01.conf fail2ban/tests/files/filter.d/testcase02.conf fail2ban/tests/files/filter.d/testcase02.local @@ -293,6 +305,7 @@ fail2ban/tests/files/logs/counter-strike fail2ban/tests/files/logs/courier-auth fail2ban/tests/files/logs/courier-smtp fail2ban/tests/files/logs/cyrus-imap +fail2ban/tests/files/logs/dante fail2ban/tests/files/logs/directadmin fail2ban/tests/files/logs/domino-smtp fail2ban/tests/files/logs/dovecot @@ -314,11 +327,15 @@ fail2ban/tests/files/logs/kerio fail2ban/tests/files/logs/lighttpd-auth fail2ban/tests/files/logs/mongodb-auth fail2ban/tests/files/logs/monit +fail2ban/tests/files/logs/monitorix +fail2ban/tests/files/logs/mssql-auth fail2ban/tests/files/logs/murmur fail2ban/tests/files/logs/mysqld-auth fail2ban/tests/files/logs/nagios fail2ban/tests/files/logs/named-refused +fail2ban/tests/files/logs/nginx-bad-request fail2ban/tests/files/logs/nginx-botsearch +fail2ban/tests/files/logs/nginx-forbidden fail2ban/tests/files/logs/nginx-http-auth fail2ban/tests/files/logs/nginx-limit-req fail2ban/tests/files/logs/nsd @@ -336,6 +353,8 @@ fail2ban/tests/files/logs/pure-ftpd fail2ban/tests/files/logs/qmail fail2ban/tests/files/logs/recidive fail2ban/tests/files/logs/roundcube-auth +fail2ban/tests/files/logs/routeros-auth +fail2ban/tests/files/logs/scanlogd fail2ban/tests/files/logs/screensharingd fail2ban/tests/files/logs/selinux-ssh fail2ban/tests/files/logs/sendmail-auth @@ -388,12 +407,12 @@ files/cacti/fail2ban_stats.sh files/cacti/README files/debian-initd files/fail2ban-logrotate +files/fail2ban-openrc.conf +files/fail2ban-openrc.init.in files/fail2ban.service.in files/fail2ban-tmpfiles.conf files/fail2ban.upstart files/gen_badbots -files/fail2ban-openrc.conf -files/fail2ban-openrc.init.in files/ipmasq-ZZZzzz_fail2ban.rul files/logwatch/fail2ban files/logwatch/fail2ban-0.8.log diff --git a/README.md b/README.md index 6bf94c25..9d5a5959 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v1.0.1.dev1 20??/??/?? + v1.1.0.dev1 20??/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -18,7 +18,7 @@ attempts, it cannot eliminate the risk presented by weak authentication. Set up services to use only two factor, or public/private authentication mechanisms if you really want to protect services. - | Since v0.10 fail2ban supports the matching of IPv6 addresses. + | Since v0.10 fail2ban supports the matching of IPv6 addresses. ------|------ This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs @@ -29,29 +29,30 @@ and the website: https://www.fail2ban.org Installation: ------------- -**It is possible that Fail2Ban is already packaged for your distribution. In -this case, you should use that instead.** +Fail2Ban is likely already packaged for your Linux distribution and [can be installed with a simple command](https://github.com/fail2ban/fail2ban/wiki/How-to-install-fail2ban-packages). + +If your distribution is not listed, you can install from GitHub: Required: -- [Python2 >= 2.7 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org) -- python-setuptools, python-distutils or python3-setuptools for installation from source +- [Python >= 3.5](https://www.python.org) or [PyPy3](https://pypy.org) +- python-setuptools (or python3-setuptools) for installation from source Optional: - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require: * Linux >= 2.6.13 -- [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) and python bindings: * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html) - [dnspython](http://www.dnspython.org/) +- [pyasyncore](https://pypi.org/project/pyasyncore/) and [pyasynchat](https://pypi.org/project/pyasynchat/) (normally bundled-in within fail2ban, for python 3.12+ only) To install: - tar xvfj fail2ban-1.0.1.tar.bz2 - cd fail2ban-1.0.1 + tar xvfj fail2ban-master.tar.bz2 + cd fail2ban-master sudo python setup.py install -Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, master or 0.11 +Alternatively, you can clone the source from GitHub to a directory of your choice, and do the install from there. Pick the correct branch, for example, master or 0.11 git clone https://github.com/fail2ban/fail2ban.git cd fail2ban @@ -90,11 +91,7 @@ fail2ban(1) and jail.conf(5) manpages for further references. Code status: ------------ -* travis-ci.org: [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=master)](https://travis-ci.org/fail2ban/fail2ban?branch=master) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.11)](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![tests status](https://secure.travis-ci.org/fail2ban/fail2ban.svg?branch=0.10)](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) - -* coveralls.io: [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=master)](https://coveralls.io/github/fail2ban/fail2ban?branch=master) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.11)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.svg?branch=0.10)](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) - -* codecov.io: [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=master)](https://codecov.io/gh/fail2ban/fail2ban/branch/master) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.11)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [![codecov.io](https://codecov.io/gh/fail2ban/fail2ban/coverage.svg?branch=0.10)](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) +* [![CI](https://github.com/fail2ban/fail2ban/actions/workflows/main.yml/badge.svg)](https://github.com/fail2ban/fail2ban/actions/workflows/main.yml) Contact: -------- diff --git a/RELEASE b/RELEASE index 2b2bc58e..96f402a0 100644 --- a/RELEASE +++ b/RELEASE @@ -13,7 +13,7 @@ Preparation * Check distribution patches and see if they can be included * https://apps.fedoraproject.org/packages/fail2ban/sources - * http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/ + * https://gitweb.gentoo.org/repo/gentoo.git/tree/net-analyzer/fail2ban * http://svnweb.freebsd.org/ports/head/security/py-fail2ban/ * https://build.opensuse.org/package/show?package=fail2ban&project=openSUSE%3AFactory * http://sophie.zarb.org/sources/fail2ban (Mageia) @@ -49,7 +49,7 @@ Preparation ad-hoc bash script to run in a clean clone: - find -type f | grep -v -e '\.git' -e '/doc/' -e '\.travis' -e MANIFEST | sed -e 's,^\./,,g' | while read f; do grep -ne "^$f\$" MANIFEST >/dev/null || echo "$f" ; done + find -type f | grep -v -e '\.git' -e '/doc/' -e MANIFEST | sed -e 's,^\./,,g' | while read f; do grep -ne "^$f\$" MANIFEST >/dev/null || echo "$f" ; done or an alternative for comparison with previous release @@ -115,7 +115,7 @@ Pre Release * Arch Linux: - * https://www.archlinux.org/packages/community/any/fail2ban/ + * https://www.archlinux.org/packages/extra/any/fail2ban/ * Debian: Yaroslav Halchenko @@ -134,7 +134,7 @@ Pre Release * Gentoo: netmon@gentoo.org - * http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/metadata.xml?view=markup + * https://gitweb.gentoo.org/repo/gentoo.git/tree/net-analyzer/fail2ban/metadata.xml * https://bugs.gentoo.org/buglist.cgi?quicksearch=fail2ban * openSUSE: Stephan Kulow diff --git a/THANKS b/THANKS index c363c76c..7c008c2c 100644 --- a/THANKS +++ b/THANKS @@ -22,6 +22,7 @@ Andrey G. Grozin Andy Fragen Arturo 'Buanzo' Busleiman Axel Thimm +Balazs Mateffy Bas van den Dikkenberg Beau Raines Bill Heaton @@ -33,6 +34,7 @@ Christoph Haas Christos Psonis craneworks Cyril Jaquier +Daniel Aleksandersen Daniel B. Cid Daniel B. Daniel Black diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 5e6843ed..31a701aa 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 09044f0a..e60d2542 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : # diff --git a/bin/fail2ban-server b/bin/fail2ban-server index 03dc0fd3..23f2fa90 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index ba3d90b9..48aae4b5 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : """Script to run Fail2Ban tests battery diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index ed958c86..d0d4a99b 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -80,7 +80,7 @@ actioncheck = # use my (Shaun's) helper PHP script by commenting out the first #actionban # line below, uncommenting the second one, and pointing the URL at # wherever you install the helper script. For the PHP helper script, see -# +# # # Tags: See jail.conf(5) man page # Values: CMD diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf index 37c42ea2..c6ce539a 100644 --- a/config/action.d/apprise.conf +++ b/config/action.d/apprise.conf @@ -10,7 +10,7 @@ # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = printf %%b "The jail as been started successfully." | -t "[Fail2Ban] : started on `uname -n`" +actionstart = printf %%b "The jail has been started successfully." | -t "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index ba6d427b..41c35497 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -30,6 +30,9 @@ [Definition] +# bypass reporting of restored (already reported) tickets: +norestored = 1 + # Option: actionstart # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 444192d3..d0029454 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -80,7 +80,7 @@ block = ip # Option: blocktype # Notes.: How to block the traffic. Use a action from man 5 ipfw # Common values: deny, unreach port, reset -# ACTION defination at the top of man ipfw for allowed values. +# ACTION definition at the top of man ipfw for allowed values. # Values: STRING # blocktype = unreach port diff --git a/config/action.d/cloudflare-token.conf b/config/action.d/cloudflare-token.conf new file mode 100644 index 00000000..dd6f2609 --- /dev/null +++ b/config/action.d/cloudflare-token.conf @@ -0,0 +1,93 @@ +# +# Author: Logic-32 +# +# IMPORTANT +# +# Please set jail.local's permission to 640 because it contains your CF API token. +# +# This action depends on curl. +# +# To get your Cloudflare API token: https://developers.cloudflare.com/api/tokens/create/ +# +# Cloudflare Firewall API: https://developers.cloudflare.com/firewall/api/cf-firewall-rules/endpoints/ + +[Definition] + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# number of failures +#